From f7de1672bb273fcf9d731384e282e5948c94d11b Mon Sep 17 00:00:00 2001 From: drew Date: Tue, 2 Jun 2026 16:48:24 -0400 Subject: [PATCH] ci: quiet CI step logs to cut Forgejo server load Apply the log-verbosity reduction from controller-coverage-optimization: each heavy step writes full output to a build/*-output.log file (still uploaded as an artifact) and streams to the live console only a one-line "OK" on success, or the COMPLETE log via `cat` on failure. The Forgejo server stores every streamed line as the live job log, and that aggregate volume across all controller-driven CI runs is what overloads it; failures remain byte-identical to before, so no diagnostic data is suppressed. Also: - apt-get update -qq / apk add -q to quiet package-manager chatter - dockerd logs redirected to a file, cat'd only on a docker build failure - paired log-artifact uploads for build/docker/helm/push-validation - TEST_PROCESSES=8 cap on unit/integration tests (prevents OOM SIGKILL / exit 137 from oversubscribing RAM on high-core hosts) The coverage gate (nox --fail-under, propagated via exit code) and the existing skip_coverage operator kill switch are preserved unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .forgejo/workflows/ci.yml | 339 +++++++++++++++++++++++++++++--------- 1 file changed, 262 insertions(+), 77 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 863aab6cd..43c512e68 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -11,6 +11,20 @@ env: PYTHON_VERSION: "3.13" NOX_DEFAULT_VENV_BACKEND: "uv" +# Logging policy (server-load reduction): +# The Forgejo server stores every line a step streams to its console as the +# live job log, and that aggregate volume across all controller-driven CI +# runs is what overloads it. The controller's CI-log fetcher +# (tools/_ci_logs.py fetch_pr_failure_logs) reads ONLY that live job log and +# ONLY to diagnose failures — it does not download artifacts. +# So each heavy step writes full output to a build/nox-*-output.log file +# (still uploaded as an artifact for humans) and streams to the console: +# - on success: a single one-line "OK" (the common path → ~zero volume), +# - on failure: the COMPLETE log via `cat` (byte-identical to today, so no +# error message or diagnostic data is suppressed). +# The COVERAGE OK/FAILED summary line and status-check result echoes are +# always streamed — the pipeline greps for them. + jobs: lint: runs-on: docker @@ -19,7 +33,7 @@ jobs: steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -35,16 +49,18 @@ jobs: restore-keys: | uv- - - name: Run lint via nox + - name: Run lint and format checks via nox run: | mkdir -p build - nox -s lint 2>&1 | tee build/nox-lint-output.log - env: - NOX_DEFAULT_VENV_BACKEND: uv - - - name: Run format check via nox - run: | - nox -s format -- --check 2>&1 | tee -a build/nox-lint-output.log + rc=0 + nox -s lint > build/nox-lint-output.log 2>&1 || rc=$? + nox -s format -- --check >> build/nox-lint-output.log 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + echo "===== lint/format FAILED (exit $rc) — full output follows; artifact ci-logs-lint =====" + cat build/nox-lint-output.log + exit "$rc" + fi + echo "lint + format: OK" env: NOX_DEFAULT_VENV_BACKEND: uv @@ -63,7 +79,7 @@ jobs: steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -82,7 +98,14 @@ jobs: - name: Run typecheck via nox run: | mkdir -p build - nox -s typecheck 2>&1 | tee build/nox-typecheck-output.log + if nox -s typecheck > build/nox-typecheck-output.log 2>&1; then + echo "typecheck: OK" + else + rc=$? + echo "===== typecheck FAILED (exit $rc) — full output follows; artifact ci-logs-typecheck =====" + cat build/nox-typecheck-output.log + exit "$rc" + fi env: NOX_DEFAULT_VENV_BACKEND: uv @@ -101,7 +124,7 @@ jobs: steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -117,16 +140,18 @@ jobs: restore-keys: | uv- - - name: Run security scan via nox + - name: Run security scan and dead-code detection via nox run: | mkdir -p build - nox -s security_scan 2>&1 | tee build/nox-security-output.log - env: - NOX_DEFAULT_VENV_BACKEND: uv - - - name: Run dead code detection via nox - run: | - nox -s dead_code 2>&1 | tee -a build/nox-security-output.log + rc=0 + nox -s security_scan > build/nox-security-output.log 2>&1 || rc=$? + nox -s dead_code >> build/nox-security-output.log 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + echo "===== security/dead_code FAILED (exit $rc) — full output follows; artifact ci-logs-security =====" + cat build/nox-security-output.log + exit "$rc" + fi + echo "security + dead_code: OK" env: NOX_DEFAULT_VENV_BACKEND: uv @@ -145,7 +170,7 @@ jobs: steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -164,7 +189,14 @@ jobs: - name: Run complexity check via nox run: | mkdir -p build - nox -s complexity 2>&1 | tee build/nox-quality-output.log + if nox -s complexity > build/nox-quality-output.log 2>&1; then + echo "complexity: OK" + else + rc=$? + echo "===== complexity FAILED (exit $rc) — full output follows; artifact ci-logs-quality =====" + cat build/nox-quality-output.log + exit "$rc" + fi env: NOX_DEFAULT_VENV_BACKEND: uv @@ -183,7 +215,7 @@ jobs: steps: - name: Install system dependencies (nodejs for checkout, git for merge tests, curl/tar for Helm) run: | - apt-get update && apt-get install -y -qq nodejs git curl tar && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs git curl tar && rm -rf /var/lib/apt/lists/* - name: Install Helm CLI run: | @@ -215,9 +247,21 @@ jobs: - name: Run unit tests via nox run: | mkdir -p build - nox -s unit_tests 2>&1 | tee build/nox-unit-tests-output.log + if nox -s unit_tests > build/nox-unit-tests-output.log 2>&1; then + echo "unit_tests: OK" + else + rc=$? + echo "===== unit_tests FAILED (exit $rc) — full output follows; artifact ci-logs-unit-tests =====" + cat build/nox-unit-tests-output.log + exit "$rc" + fi env: NOX_DEFAULT_VENV_BACKEND: uv + # behave-parallel worker cap. An uncapped host CPU + # count (e.g. 64) oversubscribes RAM and the run is + # OOM-killed mid-suite (SIGKILL / exit 137) — a red + # job with no test summary. + TEST_PROCESSES: "8" - name: Upload unit tests log artifact if: always() @@ -234,7 +278,7 @@ jobs: steps: - name: Install system dependencies (nodejs for checkout, git for integration tests, curl/tar for Helm) run: | - apt-get update && apt-get install -y -qq nodejs git curl tar && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs git curl tar && rm -rf /var/lib/apt/lists/* - name: Install Helm CLI run: | @@ -266,10 +310,21 @@ jobs: - name: Run integration tests via nox run: | mkdir -p build - nox -s integration_tests 2>&1 | tee build/nox-integration-tests-output.log + if nox -s integration_tests > build/nox-integration-tests-output.log 2>&1; then + echo "integration_tests: OK" + else + rc=$? + echo "===== integration_tests FAILED (exit $rc) — full output follows; artifact ci-logs-integration-tests =====" + cat build/nox-integration-tests-output.log + exit "$rc" + fi env: NOX_DEFAULT_VENV_BACKEND: uv CLEVERAGENTS_REQUIRE_HELM_RENDER_ASSERTIONS: "true" + # pabot worker cap. An uncapped host CPU count + # oversubscribes RAM and the run is OOM-killed + # (SIGKILL / exit 137). + TEST_PROCESSES: "8" # LLM API keys required for Robot Framework integration tests. # These secrets must be configured in Forgejo UI: # Repository Settings > Actions > Secrets @@ -315,7 +370,7 @@ jobs: - name: Install system dependencies (nodejs for checkout, git for merge tests) if: steps.gate.outputs.run == 'true' run: | - apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 if: steps.gate.outputs.run == 'true' @@ -346,8 +401,16 @@ jobs: run: | set -uo pipefail mkdir -p build - nox -s coverage_report 2>&1 | tee build/nox-coverage-output.log - rc=${PIPESTATUS[0]} + # Quiet on success; on nox failure stream the full log to the + # live console (the controller reads the streamed job log, not + # the artifact). The exit code is still propagated below, so the + # gate (nox's own --fail-under) is unchanged. + rc=0 + nox -s coverage_report > build/nox-coverage-output.log 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + echo "===== coverage_report FAILED (exit $rc) — full output follows; artifact ci-logs-coverage =====" + cat build/nox-coverage-output.log + fi # Surface the load-bearing single-line CI summary the pipeline greps. grep -E '^(nox > )?COVERAGE (OK|FAILED):' build/nox-coverage-output.log || true exit "$rc" @@ -380,7 +443,7 @@ jobs: steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -398,10 +461,26 @@ jobs: - name: Build wheel via nox run: | - nox -s build + mkdir -p build + if nox -s build > build/nox-build-output.log 2>&1; then + echo "build: OK" + else + rc=$? + echo "===== build FAILED (exit $rc) — full output follows; artifact ci-logs-build =====" + cat build/nox-build-output.log + exit "$rc" + fi env: NOX_DEFAULT_VENV_BACKEND: uv + - name: Upload build log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-logs-build + path: build/nox-build-output.log + retention-days: 30 + docker: needs: [lint, typecheck, security, quality, unit_tests] runs-on: docker @@ -411,23 +490,67 @@ jobs: steps: - name: Start Docker daemon and install dependencies run: | - dockerd & - apk add --no-cache git nodejs + # dockerd logs go to a file, not the live job log; they are + # cat'd to the console only if a docker build below fails. + dockerd > /tmp/dockerd.log 2>&1 & + apk add --no-cache -q git nodejs for i in $(seq 1 30); do docker info >/dev/null 2>&1 && break || sleep 1; done - uses: actions/checkout@v4 - name: Build Docker image (CLI) run: | - docker build -t cleverernie:test . + mkdir -p build + if docker build -t cleverernie:test . >> build/docker-output.log 2>&1; then + echo "docker build (CLI): OK" + else + rc=$? + echo "===== docker build (CLI) FAILED (exit $rc) — full output follows; artifact ci-logs-docker =====" + cat build/docker-output.log + echo "===== dockerd log =====" + cat /tmp/dockerd.log || true + exit "$rc" + fi - name: Test Docker image (CLI) run: | - docker run --rm cleverernie:test --version + mkdir -p build + if docker run --rm cleverernie:test --version >> build/docker-output.log 2>&1; then + echo "docker run (CLI --version): OK" + else + rc=$? + echo "===== docker run (CLI) FAILED (exit $rc) — full output follows; artifact ci-logs-docker =====" + cat build/docker-output.log + exit "$rc" + fi - name: Build Docker image (Server) run: | - docker build -f Dockerfile.server -t cleveragents-server:test . + mkdir -p build + if docker build -f Dockerfile.server -t cleveragents-server:test . >> build/docker-output.log 2>&1; then + echo "docker build (Server): OK" + else + rc=$? + echo "===== docker build (Server) FAILED (exit $rc) — full output follows; artifact ci-logs-docker =====" + cat build/docker-output.log + echo "===== dockerd log =====" + cat /tmp/dockerd.log || true + exit "$rc" + fi + + - name: Collect dockerd daemon log + if: always() + run: | + mkdir -p build + { echo "===== dockerd daemon log ====="; cat /tmp/dockerd.log 2>/dev/null || true; } >> build/docker-output.log + + - name: Upload docker log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-logs-docker + path: build/docker-output.log + retention-days: 30 helm: runs-on: docker @@ -436,7 +559,7 @@ jobs: steps: - name: Install system dependencies (nodejs for checkout, curl for Helm and kubeconform) run: | - apt-get update && apt-get install -y -qq nodejs curl tar && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs curl tar && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 @@ -467,27 +590,67 @@ jobs: - name: Build Helm chart dependencies run: | - helm dependency build ./k8s + mkdir -p build + if helm dependency build ./k8s >> build/helm-output.log 2>&1; then + echo "helm dependency build: OK" + else + rc=$? + echo "===== helm dependency build FAILED (exit $rc) — full output follows; artifact ci-logs-helm =====" + cat build/helm-output.log + exit "$rc" + fi - name: Helm lint chart run: | - helm lint ./k8s \ - --set database.url="postgresql+asyncpg://user:pass@db-host:5432/cleveragents" + mkdir -p build + if helm lint ./k8s \ + --set database.url="postgresql+asyncpg://user:pass@db-host:5432/cleveragents" >> build/helm-output.log 2>&1; then + echo "helm lint: OK" + else + rc=$? + echo "===== helm lint FAILED (exit $rc) — full output follows; artifact ci-logs-helm =====" + cat build/helm-output.log + exit "$rc" + fi - name: Helm template smoke render run: | - helm template cleveragents ./k8s \ - --set database.url="postgresql+asyncpg://user:pass@db-host:5432/cleveragents" >/tmp/rendered.yaml - test -s /tmp/rendered.yaml + mkdir -p build + if helm template cleveragents ./k8s \ + --set database.url="postgresql+asyncpg://user:pass@db-host:5432/cleveragents" >/tmp/rendered.yaml 2>> build/helm-output.log \ + && test -s /tmp/rendered.yaml; then + echo "helm template: OK" + else + rc=$? + echo "===== helm template FAILED (exit $rc) — full output follows; artifact ci-logs-helm =====" + cat build/helm-output.log + exit "$rc" + fi - name: Validate rendered manifests with kubeconform run: | - kubeconform \ + mkdir -p build + if kubeconform \ -strict \ -ignore-missing-schemas \ -kubernetes-version 1.29.0 \ -summary \ - /tmp/rendered.yaml + /tmp/rendered.yaml >> build/helm-output.log 2>&1; then + echo "kubeconform: OK" + else + rc=$? + echo "===== kubeconform FAILED (exit $rc) — full output follows; artifact ci-logs-helm =====" + cat build/helm-output.log + exit "$rc" + fi + + - name: Upload helm log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-logs-helm + path: build/helm-output.log + retention-days: 30 push-validation: @@ -501,7 +664,7 @@ jobs: steps: - name: Install system dependencies (nodejs for checkout, git for push validation) run: | - apt-get update && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/* + apt-get update -qq && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/* - name: Checkout with explicit write credentials uses: actions/checkout@v4 @@ -527,16 +690,20 @@ jobs: # Confirm that the credential helper set up by actions/checkout # is active. This ensures HTTPS push operations will authenticate # correctly without prompting for a password. - echo "=== Git credential configuration ===" - git config --list | grep -E "credential|url" || echo "WARNING: No credential helper found" - echo "=== Remote URL ===" - git remote get-url origin - echo "=== Credential helper check ===" - if git config credential.helper > /dev/null 2>&1; then - echo "OK: Credential helper is configured: $(git config credential.helper)" - else - echo "WARNING: No credential helper configured — push may fail" - fi + mkdir -p build + { + echo "=== Git credential configuration ===" + git config --list | grep -E "credential|url" || echo "WARNING: No credential helper found" + echo "=== Remote URL ===" + git remote get-url origin + echo "=== Credential helper check ===" + if git config credential.helper > /dev/null 2>&1; then + echo "OK: Credential helper is configured: $(git config credential.helper)" + else + echo "WARNING: No credential helper configured — push may fail" + fi + } >> build/push-validation-output.log 2>&1 + echo "credential-helper check: done" - name: Smoke-test push access via API # Validates write permission using the Forgejo API before attempting @@ -546,28 +713,46 @@ jobs: FORGEJO_URL: ${{ secrets.FORGEJO_URL }} FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} run: | - REPO="${{ forgejo.repository }}" - API_URL="${FORGEJO_URL}/api/v1/repos/${REPO}" - echo "=== Testing repository API access ===" - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token ${FORGEJO_TOKEN}" \ - "${API_URL}") - if [ "${HTTP_STATUS}" != "200" ]; then - echo "ERROR: FORGEJO_TOKEN cannot access repository API (HTTP ${HTTP_STATUS})." - echo "Ensure FORGEJO_TOKEN is set in Repository Settings > Actions > Secrets" - echo "and that the token has repository (write) scope." - exit 1 + mkdir -p build + rc=0 + ( + REPO="${{ forgejo.repository }}" + API_URL="${FORGEJO_URL}/api/v1/repos/${REPO}" + echo "=== Testing repository API access ===" + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + "${API_URL}") + if [ "${HTTP_STATUS}" != "200" ]; then + echo "ERROR: FORGEJO_TOKEN cannot access repository API (HTTP ${HTTP_STATUS})." + echo "Ensure FORGEJO_TOKEN is set in Repository Settings > Actions > Secrets" + echo "and that the token has repository (write) scope." + exit 1 + fi + PUSH_ALLOWED=$(curl -s \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + "${API_URL}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(str(d.get('permissions',{}).get('push',False)).lower())") + if [ "${PUSH_ALLOWED}" != "true" ]; then + echo "ERROR: FORGEJO_TOKEN does not have push (write) permission." + echo "Grant the token Contents: Write permission or use a token with full repository scope." + exit 1 + fi + echo "OK: Push access verified -- FORGEJO_TOKEN has write permission on ${REPO}" + echo "=== Push access smoke-test passed ===" + ) >> build/push-validation-output.log 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + echo "===== push-validation FAILED (exit $rc) — full output follows; artifact ci-logs-push-validation =====" + cat build/push-validation-output.log + exit "$rc" fi - PUSH_ALLOWED=$(curl -s \ - -H "Authorization: token ${FORGEJO_TOKEN}" \ - "${API_URL}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(str(d.get('permissions',{}).get('push',False)).lower())") - if [ "${PUSH_ALLOWED}" != "true" ]; then - echo "ERROR: FORGEJO_TOKEN does not have push (write) permission." - echo "Grant the token Contents: Write permission or use a token with full repository scope." - exit 1 - fi - echo "OK: Push access verified -- FORGEJO_TOKEN has write permission on ${REPO}" - echo "=== Push access smoke-test passed ===" + echo "push-validation: OK" + + - name: Upload push-validation log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-logs-push-validation + path: build/push-validation-output.log + retention-days: 30 status-check: if: always() needs: [lint, typecheck, security, quality, unit_tests, integration_tests, coverage, build, docker, helm, push-validation] -- 2.52.0