chore(agents): add milestone-based PR prioritization to ca-continuous-pr-reviewer #10835

Merged
HAL9000 merged 4 commits from feature/m3111-milestone-based-pr-prioritization into master 2026-06-06 16:10:32 +00:00
3 changed files with 404 additions and 0 deletions
@@ -0,0 +1,77 @@
---
description: >
Continuous PR reviewer that prioritizes pull requests based on their
associated milestone. Reviews PRs in milestone order, ensuring critical
path items are reviewed first. Processes multiple PRs in a single session
and exits when all eligible PRs have been reviewed.
mode: subagent
hidden: true
temperature: 0.2
model: anthropic/claude-haiku-4-5
reasoningEffort: "max"
color: warning
permission:
"*": deny
"doom_loop": deny
question: deny
"sequential-thinking*": allow
edit: deny
webfetch: deny
bash:
"*": deny
"curl *": allow
task:
"*": deny
"ci-log-fetcher": allow
"forgejo-label-manager": allow
"forgejo_*": deny
"forgejo_get_issue_by_index": allow
"forgejo_list_issue_comments": allow
"forgejo_get_pull_request_by_index": allow
"forgejo_get_pull_request_diff": allow
"forgejo_list_pull_request_files": allow
"forgejo_list_pull_reviews": allow
"forgejo_list_pull_review_comments": allow
"forgejo_list_repo_milestones": allow
"forgejo_list_repo_pull_requests": allow
"forgejo_get_file_content": allow
---
# Continuous PR Reviewer
You review multiple pull requests in a single session, prioritizing them based on their
associated milestone. You process PRs in milestone order, ensuring critical path items
(Must Have, Should Have) are reviewed before lower-priority items. You exit when all
eligible PRs have been reviewed.
## Milestone-Based Prioritization
1. Fetch all milestones using forgejo_list_repo_milestones (paginate fully)
2. Fetch all open PRs using forgejo_list_repo_pull_requests with state=open (paginate fully)
3. Assign priority scores to each PR based on:
- Milestone due date: earlier due dates = higher priority
- MoSCoW label: Must Have (100) > Should Have (50) > Could Have (10) > no label (0)
- PR age: older PRs = slightly higher priority (tie-breaker)
4. Sort PRs by priority score (descending)
5. Review PRs in order until all are processed or session limit reached
## **CRITICAL** Rules
1. Process all eligible PRs. Do not exit early unless session limit is reached.
2. Milestone-based ordering is mandatory. Always sort by milestone priority before reviewing.
3. Use reviewer credentials for all writes. Never post reviews as the primary bot.
4. Always post both a formal review AND a backup comment.
5. Never fix code or merge PRs. You only review.
6. Apply labels via forgejo-label-manager. Never apply labels directly.
7. Exhaustive pagination for all list results. Always set limit to 50 and paginate fully.
## Milestone Priority Calculation
priority_score = (milestone_weight * 1000) + (moscow_weight * 100) + (age_weight)
where:
milestone_weight = (max_due_date - pr_milestone_due_date) / (max_due_date - min_due_date)
moscow_weight = { "Must Have": 100, "Should Have": 50, "Could Have": 10, "no label": 0 }
age_weight = (current_time - pr_created_at) / 86400
Higher scores = higher priority = review first.
@@ -0,0 +1,24 @@
Feature: Continuous PR Reviewer with Milestone-Based Prioritization
As a code reviewer
I want to review multiple PRs in a single session
So that I can prioritize reviews based on milestone criticality
Scenario: Prioritize PRs by milestone due date
Given a repository with multiple open PRs
And PRs assigned to different milestones
When the continuous PR reviewer starts
Then PRs are reviewed in milestone priority order
And earlier milestone due dates are reviewed first
Scenario: Prioritize by MoSCoW labels within milestone
Given PRs in the same milestone
And some PRs have MoSCoW labels
When the continuous PR reviewer processes them
Then Must Have PRs are reviewed before Should Have
And Should Have PRs are reviewed before Could Have
Scenario: Use PR age as tie-breaker
Given multiple PRs with same milestone and MoSCoW label
When the continuous PR reviewer sorts them
Then older PRs are reviewed first
And newer PRs are reviewed last
@@ -0,0 +1,303 @@
"""Step definitions for Continuous PR Reviewer with Milestone-Based Prioritization.
Tests the priority scoring algorithm described in the ca-continuous-pr-reviewer
agent specification:
priority_score = (milestone_weight * 1000) + (moscow_weight * 100) + (age_weight)
where:
milestone_weight = (max_due_date - pr_milestone_due_date) / (max_due_date - min_due_date)
moscow_weight = { "Must Have": 100, "Should Have": 50, "Could Have": 10, "no label": 0 }
age_weight = (current_time - pr_created_at) / 86400
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from behave import given, then, when
from behave.runner import Context
_MOSCOW_WEIGHTS: dict[str, int] = {
"Must Have": 100,
"Should Have": 50,
"Could Have": 10,
"no label": 0,
}
def _compute_priority_score(
pr: dict[str, Any],
min_due_ts: float,
max_due_ts: float,
now_ts: float,
) -> float:
"""Compute the priority score for a single PR."""
due_ts: float | None = pr.get("milestone_due_ts")
if due_ts is None or max_due_ts == min_due_ts:
milestone_weight: float = 0.0
else:
milestone_weight = (max_due_ts - due_ts) / (max_due_ts - min_due_ts)
moscow_label: str = pr.get("moscow_label", "no label")
moscow_weight: int = _MOSCOW_WEIGHTS.get(moscow_label, 0)
created_ts: float = pr.get("created_ts", now_ts)
age_weight: float = (now_ts - created_ts) / 86400.0
return (milestone_weight * 1000.0) + (moscow_weight * 100.0) + age_weight
def _sort_prs_by_priority(
prs: list[dict[str, Any]],
now_ts: float | None = None,
) -> list[dict[str, Any]]:
"""Return *prs* sorted by descending priority score."""
import time
Outdated
Review

BLOCKING — Python import rule violation

import time is inside the function body of _sort_prs_by_priority(). Per CONTRIBUTING.md: "Python: all at top, from X import Y, if TYPE_CHECKING: only exception." All imports must be at the top of the file.

How to fix: Move import time to the top-level import section alongside the other imports (from __future__ import annotations, from datetime import UTC, datetime, etc.).

Example fix:

from __future__ import annotations

import time
from datetime import UTC, datetime
from typing import Any

from behave import given, then, when
from behave.runner import Context

Then remove the import time line from inside _sort_prs_by_priority().


Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker

**BLOCKING — Python import rule violation** `import time` is inside the function body of `_sort_prs_by_priority()`. Per CONTRIBUTING.md: *"Python: all at top, `from X import Y`, `if TYPE_CHECKING:` only exception."* All imports must be at the top of the file. **How to fix:** Move `import time` to the top-level import section alongside the other imports (`from __future__ import annotations`, `from datetime import UTC, datetime`, etc.). Example fix: ```python from __future__ import annotations import time from datetime import UTC, datetime from typing import Any from behave import given, then, when from behave.runner import Context ``` Then remove the `import time` line from inside `_sort_prs_by_priority()`. --- Automated by CleverAgents Bot Supervisor: PR Review | Agent: pr-review-worker
if now_ts is None:
now_ts = time.time()
due_timestamps: list[float] = [
pr["milestone_due_ts"] for pr in prs if pr.get("milestone_due_ts") is not None
]
min_due = min(due_timestamps) if due_timestamps else 0.0
max_due = max(due_timestamps) if due_timestamps else 0.0
scored = [(pr, _compute_priority_score(pr, min_due, max_due, now_ts)) for pr in prs]
scored.sort(key=lambda x: x[1], reverse=True)
return [pr for pr, _ in scored]
def _make_pr(
pr_id: int,
title: str,
milestone_due: str | None = None,
moscow_label: str = "no label",
created: str | None = None,
) -> dict[str, Any]:
"""Build a minimal PR dict for testing."""
due_ts: float | None = None
if milestone_due is not None:
due_ts = datetime.fromisoformat(milestone_due).replace(tzinfo=UTC).timestamp()
created_ts: float = 0.0
if created is not None:
created_ts = datetime.fromisoformat(created).replace(tzinfo=UTC).timestamp()
return {
"id": pr_id,
"title": title,
"milestone_due_ts": due_ts,
"moscow_label": moscow_label,
"created_ts": created_ts,
}
@given("a repository with multiple open PRs")
def step_given_repository_with_prs(context: Context) -> None:
"""Initialise an empty PR list on the context."""
context.prs: list[dict[str, Any]] = []
context.sorted_prs: list[dict[str, Any]] = []
context.now_ts: float = datetime(2026, 1, 1, tzinfo=UTC).timestamp()
@given("PRs assigned to different milestones")
def step_given_prs_with_different_milestones(context: Context) -> None:
"""Populate the PR list with PRs that have different milestone due dates."""
context.prs = [
_make_pr(
1,
"PR for milestone v1.0 (earliest)",
milestone_due="2026-02-01",
created="2025-12-01",
),
_make_pr(
2,
"PR for milestone v2.0 (middle)",
milestone_due="2026-04-01",
created="2025-12-05",
),
_make_pr(
3,
"PR for milestone v3.0 (latest)",
milestone_due="2026-06-01",
created="2025-12-10",
),
]
@given("PRs in the same milestone")
def step_given_prs_in_same_milestone(context: Context) -> None:
"""Populate the PR list with PRs sharing the same milestone."""
context.prs = []
context.sorted_prs = []
context.now_ts = datetime(2026, 1, 1, tzinfo=UTC).timestamp()
context.prs = [
_make_pr(
10,
"Must Have PR",
milestone_due="2026-03-01",
moscow_label="Must Have",
created="2025-12-01",
),
_make_pr(
11,
"Should Have PR",
milestone_due="2026-03-01",
moscow_label="Should Have",
created="2025-12-01",
),
_make_pr(
12,
"Could Have PR",
milestone_due="2026-03-01",
moscow_label="Could Have",
created="2025-12-01",
),
]
@given("some PRs have MoSCoW labels")
def step_given_prs_have_moscow_labels(context: Context) -> None:
"""No-op: MoSCoW labels are already set in the previous Given step."""
pass
@given("multiple PRs with same milestone and MoSCoW label")
def step_given_prs_same_milestone_and_moscow(context: Context) -> None:
"""Populate the PR list with PRs sharing milestone and MoSCoW label."""
context.prs = []
context.sorted_prs = []
context.now_ts = datetime(2026, 1, 1, tzinfo=UTC).timestamp()
context.prs = [
_make_pr(
20,
"Newest PR (lowest age weight)",
milestone_due="2026-03-01",
moscow_label="Must Have",
created="2025-12-20",
),
_make_pr(
21,
"Middle-aged PR",
milestone_due="2026-03-01",
moscow_label="Must Have",
created="2025-12-10",
),
_make_pr(
22,
"Oldest PR (highest age weight)",
milestone_due="2026-03-01",
moscow_label="Must Have",
created="2025-11-01",
),
]
@when("the continuous PR reviewer starts")
def step_when_reviewer_starts(context: Context) -> None:
"""Sort the PR list using the milestone-based priority algorithm."""
context.sorted_prs = _sort_prs_by_priority(context.prs, now_ts=context.now_ts)
@when("the continuous PR reviewer processes them")
def step_when_reviewer_processes(context: Context) -> None:
"""Sort the PR list using the milestone-based priority algorithm."""
context.sorted_prs = _sort_prs_by_priority(context.prs, now_ts=context.now_ts)
@when("the continuous PR reviewer sorts them")
def step_when_reviewer_sorts(context: Context) -> None:
"""Sort the PR list using the milestone-based priority algorithm."""
context.sorted_prs = _sort_prs_by_priority(context.prs, now_ts=context.now_ts)
@then("PRs are reviewed in milestone priority order")
def step_then_prs_in_milestone_priority_order(context: Context) -> None:
"""Assert that the sorted PR list is non-empty and ordered."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
assert len(sorted_prs) == len(context.prs), (
f"Expected {len(context.prs)} PRs after sorting, got {len(sorted_prs)}"
)
@then("earlier milestone due dates are reviewed first")
def step_then_earlier_milestones_first(context: Context) -> None:
"""Assert that PRs with earlier milestone due dates appear first."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
due_timestamps: list[float] = [
pr["milestone_due_ts"]
for pr in sorted_prs
if pr.get("milestone_due_ts") is not None
]
for i in range(len(due_timestamps) - 1):
assert due_timestamps[i] <= due_timestamps[i + 1], (
f"Expected PR at position {i} (due={due_timestamps[i]}) to have "
f"an earlier or equal due date than PR at position {i + 1} "
f"(due={due_timestamps[i + 1]}). "
f"Sorted PR titles: {[p['title'] for p in sorted_prs]}"
)
@then("Must Have PRs are reviewed before Should Have")
def step_then_must_have_before_should_have(context: Context) -> None:
"""Assert that Must Have PRs appear before Should Have PRs."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
must_have_indices = [
i for i, pr in enumerate(sorted_prs) if pr.get("moscow_label") == "Must Have"
]
should_have_indices = [
i for i, pr in enumerate(sorted_prs) if pr.get("moscow_label") == "Should Have"
]
if must_have_indices and should_have_indices:
assert max(must_have_indices) < min(should_have_indices), (
f"Expected all Must Have PRs (positions {must_have_indices}) to appear "
f"before all Should Have PRs (positions {should_have_indices}). "
f"Sorted PR labels: {[p['moscow_label'] for p in sorted_prs]}"
)
@then("Should Have PRs are reviewed before Could Have")
def step_then_should_have_before_could_have(context: Context) -> None:
"""Assert that Should Have PRs appear before Could Have PRs."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
should_have_indices = [
i for i, pr in enumerate(sorted_prs) if pr.get("moscow_label") == "Should Have"
]
could_have_indices = [
i for i, pr in enumerate(sorted_prs) if pr.get("moscow_label") == "Could Have"
]
if should_have_indices and could_have_indices:
assert max(should_have_indices) < min(could_have_indices), (
f"Expected all Should Have PRs (positions {should_have_indices}) to appear "
f"before all Could Have PRs (positions {could_have_indices}). "
f"Sorted PR labels: {[p['moscow_label'] for p in sorted_prs]}"
)
@then("older PRs are reviewed first")
def step_then_older_prs_first(context: Context) -> None:
"""Assert that older PRs (earlier created_ts) appear first in the sorted list."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
created_timestamps: list[float] = [pr["created_ts"] for pr in sorted_prs]
for i in range(len(created_timestamps) - 1):
assert created_timestamps[i] <= created_timestamps[i + 1], (
f"Expected PR at position {i} (created={created_timestamps[i]}) to be "
f"older than or equal to PR at position {i + 1} "
f"(created={created_timestamps[i + 1]}). "
f"Sorted PR titles: {[p['title'] for p in sorted_prs]}"
)
@then("newer PRs are reviewed last")
def step_then_newer_prs_last(context: Context) -> None:
"""Assert that the last PR in the sorted list is the newest one."""
sorted_prs = context.sorted_prs
assert sorted_prs, "Expected a non-empty sorted PR list"
last_pr = sorted_prs[-1]
max_created_ts = max(pr["created_ts"] for pr in sorted_prs)
assert last_pr["created_ts"] == max_created_ts, (
f"Expected the last PR to be the newest (created_ts={max_created_ts}), "
f"but got created_ts={last_pr['created_ts']} for PR '{last_pr['title']}'. "
f"Sorted PR titles: {[p['title'] for p in sorted_prs]}"
)