feat(observability): implement Metrics Collection Framework (14 metric types with Histogram/Counter/Gauge) #672

Merged
freemo merged 1 commits from feature/m6-metrics-collection-framework into master 2026-03-10 18:16:29 +00:00
13 changed files with 1490 additions and 5 deletions
+101
View File
@@ -0,0 +1,101 @@
"""ASV benchmarks for metrics collection framework overhead.
Measures the time to create metric entries via histogram/counter/gauge
factory methods, exercise all 14 convenience methods, and emit metrics
through the MetricsEmitter.
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
from unittest.mock import MagicMock
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.domain.models.observability.metrics import ( # noqa: E402
MetricCollector,
MetricType,
OperationalMetricKey,
)
from cleveragents.infrastructure.observability.metrics_emitter import ( # noqa: E402
MetricsEmitter,
)
VALID_PLAN_ID = "01HX0000000000MMMMMMMMMMMM"
class MetricCollectorSuite:
"""Benchmark MetricCollector typed factory methods."""
def time_histogram(self) -> None:
"""Create a histogram metric entry."""
MetricCollector.histogram(
OperationalMetricKey.PLAN_DURATION_MS,
1500.0,
VALID_PLAN_ID,
)
def time_counter(self) -> None:
"""Create a counter metric entry."""
MetricCollector.counter(
OperationalMetricKey.LLM_CALL_COUNT,
5.0,
VALID_PLAN_ID,
)
def time_gauge(self) -> None:
"""Create a gauge metric entry."""
MetricCollector.gauge(
OperationalMetricKey.PLAN_DECISION_COUNT,
3.0,
VALID_PLAN_ID,
)
def time_all_14_convenience_methods(self) -> None:
"""Exercise all 14 convenience methods."""
MetricCollector.plan_duration(VALID_PLAN_ID, 1000.0)
MetricCollector.plan_cost(VALID_PLAN_ID, 0.05)
MetricCollector.plan_decision_count(VALID_PLAN_ID, 5.0)
MetricCollector.subplan_count(VALID_PLAN_ID, 2.0)
MetricCollector.actor_invocation_count(VALID_PLAN_ID, 1.0)
MetricCollector.actor_latency(VALID_PLAN_ID, 250.0)
MetricCollector.tool_invocation_count(VALID_PLAN_ID, 10.0)
MetricCollector.tool_error_rate(VALID_PLAN_ID, 0.01)
MetricCollector.context_build_time(VALID_PLAN_ID, 50.0)
MetricCollector.context_token_count(VALID_PLAN_ID, 8000.0)
MetricCollector.llm_call_count(VALID_PLAN_ID, 3.0)
MetricCollector.llm_total_tokens(VALID_PLAN_ID, 5000.0)
MetricCollector.llm_total_cost(VALID_PLAN_ID, 0.5)
MetricCollector.llm_avg_latency(VALID_PLAN_ID, 200.0)
class MetricsEmitterSuite:
"""Benchmark MetricsEmitter operations."""
def setup(self) -> None:
self._emitter = MetricsEmitter(enabled=True)
self._entries = [
MetricCollector.plan_duration(VALID_PLAN_ID, float(i * 100))
for i in range(100)
]
def time_emit_single(self) -> None:
"""Emit a single metric entry."""
self._emitter.emit(self._entries[0])
def time_emit_batch_100(self) -> None:
"""Emit 100 metric entries in a batch."""
self._emitter.emit_batch(self._entries)
def time_emit_disabled_noop(self) -> None:
"""Verify disabled emitter is fast no-op."""
disabled = MetricsEmitter(enabled=False)
disabled.emit_batch(self._entries)
@@ -0,0 +1,196 @@
Feature: Metrics Collection Framework
As a platform operator
I want structured metric collection with typed semantics
So that I can monitor plan execution via histograms, counters, and gauges
# --- MetricType enum ---
Scenario: MetricType enum has three members
Then MetricType should have exactly 3 members
Scenario: MetricType values are correct
Then MetricType HISTOGRAM should equal "histogram"
And MetricType COUNTER should equal "counter"
And MetricType GAUGE should equal "gauge"
# --- MetricDefinition registry ---
Scenario: All 14 metric keys have definitions
Then METRIC_DEFINITIONS should have exactly 14 entries
Scenario: PLAN_DURATION_MS is a histogram
Then the definition for PLAN_DURATION_MS should have metric_type HISTOGRAM
Scenario: LLM_CALL_COUNT is a counter
Then the definition for LLM_CALL_COUNT should have metric_type COUNTER
Scenario: PLAN_DECISION_COUNT is a gauge
Then the definition for PLAN_DECISION_COUNT should have metric_type GAUGE
Scenario: TOOL_ERROR_RATE is a gauge
Then the definition for TOOL_ERROR_RATE should have metric_type GAUGE
# --- MetricCollector typed factory methods ---
Scenario: histogram method creates a metric with histogram type
Given a plan_id for metrics collection
When I create a histogram metric for PLAN_DURATION_MS with value 1200.0
Then the result metric_type should be HISTOGRAM
And the result value should be 1200.0
Scenario: counter method creates a metric with counter type
Given a plan_id for metrics collection
When I create a counter metric for LLM_CALL_COUNT with value 5.0
Then the result metric_type should be COUNTER
And the result value should be 5.0
Scenario: gauge method creates a metric with gauge type
Given a plan_id for metrics collection
When I create a gauge metric for PLAN_DECISION_COUNT with value 3.0
Then the result metric_type should be GAUGE
And the result value should be 3.0
# --- All 14 convenience methods ---
Scenario: plan_duration convenience method
Given a plan_id for metrics collection
When I use the plan_duration convenience with value 2500.0
Then the collected metric key should equal PLAN_DURATION_MS
And the collected metric value should equal 2500.0
And the metric_type should be HISTOGRAM
Scenario: plan_cost convenience method
Given a plan_id for metrics collection
When I use the plan_cost convenience with value 0.12
Then the collected metric key should equal PLAN_TOTAL_COST_USD
And the collected metric value should equal 0.12
Scenario: plan_decision_count convenience method
Given a plan_id for metrics collection
When I use the plan_decision_count convenience with value 7.0
Then the collected metric key should equal PLAN_DECISION_COUNT
And the collected metric value should equal 7.0
Scenario: subplan_count convenience method
Given a plan_id for metrics collection
When I use the subplan_count convenience with value 2.0
Then the collected metric key should equal SUBPLAN_COUNT
And the collected metric value should equal 2.0
Scenario: actor_invocation_count convenience method
Given a plan_id for metrics collection
When I use the actor_invocation_count convenience with value 1.0
Then the collected metric key should equal ACTOR_INVOCATION_COUNT
And the collected metric value should equal 1.0
Scenario: actor_latency convenience method
Given a plan_id for metrics collection
When I use the actor_latency convenience with value 350.0
Then the collected metric key should equal ACTOR_LATENCY_MS
And the collected metric value should equal 350.0
Scenario: tool_invocation_count convenience method
Given a plan_id for metrics collection
When I use the tool_invocation_count convenience with value 10.0
Then the collected metric key should equal TOOL_INVOCATION_COUNT
And the collected metric value should equal 10.0
Scenario: tool_error_rate convenience method
Given a plan_id for metrics collection
When I use the tool_error_rate convenience with value 0.05
Then the collected metric key should equal TOOL_ERROR_RATE
And the collected metric value should equal 0.05
Scenario: context_build_time convenience method
Given a plan_id for metrics collection
When I use the context_build_time convenience with value 45.0
Then the collected metric key should equal CONTEXT_BUILD_TIME_MS
And the collected metric value should equal 45.0
Scenario: context_token_count convenience method
Given a plan_id for metrics collection
When I use the context_token_count convenience with value 8192.0
Then the collected metric key should equal CONTEXT_TOKEN_COUNT
And the collected metric value should equal 8192.0
Scenario: llm_call_count convenience method
Given a plan_id for metrics collection
When I use the llm_call_count convenience with value 15.0
Then the collected metric key should equal LLM_CALL_COUNT
And the collected metric value should equal 15.0
Scenario: llm_total_tokens convenience method
Given a plan_id for metrics collection
When I use the llm_total_tokens convenience with value 50000.0
Then the collected metric key should equal LLM_TOTAL_TOKENS
And the collected metric value should equal 50000.0
Scenario: llm_total_cost convenience method
Given a plan_id for metrics collection
When I use the llm_total_cost convenience with value 1.25
Then the collected metric key should equal LLM_TOTAL_COST_USD
And the collected metric value should equal 1.25
Scenario: llm_avg_latency convenience method
Given a plan_id for metrics collection
When I use the llm_avg_latency convenience with value 200.0
Then the collected metric key should equal LLM_AVG_LATENCY_MS
And the collected metric value should equal 200.0
# --- MetricsEmitter ---
Scenario: MetricsEmitter emits metric as structured log
Given a plan_id for metrics collection
And a MetricsEmitter in local mode
When I emit a plan duration metric with value 1500.0
Then the emitter should have emitted successfully
Scenario: MetricsEmitter respects disabled setting
Given a plan_id for metrics collection
And a MetricsEmitter that is disabled
When I emit a plan duration metric with value 1500.0
Then the emitter should not have emitted
Scenario: MetricsEmitter emits batch
Given a plan_id for metrics collection
And a MetricsEmitter in local mode
When I emit a batch of 3 metrics
Then the emitter should report 3 emitted
Scenario: MetricsEmitter disabled batch returns zero
Given a plan_id for metrics collection
And a MetricsEmitter that is disabled
When I emit a batch of 3 metrics
Then the emitter should report 0 emitted
Scenario: MetricsEmitter from_settings uses defaults
When I create a MetricsEmitter from default settings
Then the emitter should be enabled
And the emitter prometheus should be disabled
# --- Configuration ---
Scenario: metrics_enabled defaults to True
When I load default settings for metrics
Then metrics_enabled should be True
Scenario: metrics_export_prometheus defaults to False
When I load default settings for metrics
Then metrics_export_prometheus should be False
# --- Metrics processor ---
Scenario: metrics_log_processor tags metric events
When I pass a metric event through the metrics processor
Then the event should have event_category metric
Scenario: metrics_log_processor ignores non-metric events
When I pass a non-metric event through the metrics processor
Then the event should not have event_category
# --- MetricEntry includes metric_type ---
Scenario: MetricEntry from record includes resolved metric_type
Given a plan_id for metrics collection
When I record a metric via collector record for ACTOR_LATENCY_MS
Then the entry metric_type should be HISTOGRAM
+357
View File
@@ -0,0 +1,357 @@
"""Step definitions for the metrics collection framework feature."""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from cleveragents.config.metrics_processor import metrics_log_processor
from cleveragents.config.settings import Settings
from cleveragents.domain.models.observability.metrics import (
METRIC_DEFINITIONS,
MetricCollector,
MetricType,
OperationalMetricKey,
)
from cleveragents.infrastructure.observability.metrics_emitter import MetricsEmitter
VALID_PLAN_ID = "01HX0000000000MMMMMMMMMMMM"
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given("a plan_id for metrics collection")
def step_given_plan_id(context: Context) -> None:
context.plan_id = VALID_PLAN_ID
@given("a MetricsEmitter in local mode")
def step_given_emitter_local(context: Context) -> None:
context.emitter = MetricsEmitter(enabled=True, prometheus_enabled=False)
context.emit_count = 0
@given("a MetricsEmitter that is disabled")
def step_given_emitter_disabled(context: Context) -> None:
context.emitter = MetricsEmitter(enabled=False)
context.emit_count = 0
# ---------------------------------------------------------------------------
# When steps — typed factory methods
# ---------------------------------------------------------------------------
@when("I create a histogram metric for {key} with value {value}")
def step_when_histogram(context: Context, key: str, value: str) -> None:
metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
context.result_entry = MetricCollector.histogram(
metric_key, float(value), context.plan_id
)
@when("I create a counter metric for {key} with value {value}")
def step_when_counter(context: Context, key: str, value: str) -> None:
metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
context.result_entry = MetricCollector.counter(
metric_key, float(value), context.plan_id
)
@when("I create a gauge metric for {key} with value {value}")
def step_when_gauge(context: Context, key: str, value: str) -> None:
metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
context.result_entry = MetricCollector.gauge(
metric_key, float(value), context.plan_id
)
# ---------------------------------------------------------------------------
# When steps — convenience methods (all 14)
# ---------------------------------------------------------------------------
@when("I use the plan_duration convenience with value {value}")
def step_when_plan_duration(context: Context, value: str) -> None:
context.result_entry = MetricCollector.plan_duration(context.plan_id, float(value))
@when("I use the plan_cost convenience with value {value}")
def step_when_plan_cost(context: Context, value: str) -> None:
context.result_entry = MetricCollector.plan_cost(context.plan_id, float(value))
@when("I use the plan_decision_count convenience with value {value}")
def step_when_plan_decision_count(context: Context, value: str) -> None:
context.result_entry = MetricCollector.plan_decision_count(
context.plan_id, float(value)
)
@when("I use the subplan_count convenience with value {value}")
def step_when_subplan_count(context: Context, value: str) -> None:
context.result_entry = MetricCollector.subplan_count(context.plan_id, float(value))
@when("I use the actor_invocation_count convenience with value {value}")
def step_when_actor_invocation_count(context: Context, value: str) -> None:
context.result_entry = MetricCollector.actor_invocation_count(
context.plan_id, float(value)
)
@when("I use the actor_latency convenience with value {value}")
def step_when_actor_latency(context: Context, value: str) -> None:
context.result_entry = MetricCollector.actor_latency(context.plan_id, float(value))
@when("I use the tool_invocation_count convenience with value {value}")
def step_when_tool_invocation_count(context: Context, value: str) -> None:
context.result_entry = MetricCollector.tool_invocation_count(
context.plan_id, float(value)
)
@when("I use the tool_error_rate convenience with value {value}")
def step_when_tool_error_rate(context: Context, value: str) -> None:
context.result_entry = MetricCollector.tool_error_rate(
context.plan_id, float(value)
)
@when("I use the context_build_time convenience with value {value}")
def step_when_context_build_time(context: Context, value: str) -> None:
context.result_entry = MetricCollector.context_build_time(
context.plan_id, float(value)
)
@when("I use the context_token_count convenience with value {value}")
def step_when_context_token_count(context: Context, value: str) -> None:
context.result_entry = MetricCollector.context_token_count(
context.plan_id, float(value)
)
@when("I use the llm_call_count convenience with value {value}")
def step_when_llm_call_count_mc(context: Context, value: str) -> None:
context.result_entry = MetricCollector.llm_call_count(context.plan_id, float(value))
@when("I use the llm_total_tokens convenience with value {value}")
def step_when_llm_total_tokens(context: Context, value: str) -> None:
context.result_entry = MetricCollector.llm_total_tokens(
context.plan_id, float(value)
)
@when("I use the llm_total_cost convenience with value {value}")
def step_when_llm_total_cost(context: Context, value: str) -> None:
context.result_entry = MetricCollector.llm_total_cost(context.plan_id, float(value))
@when("I use the llm_avg_latency convenience with value {value}")
def step_when_llm_avg_latency(context: Context, value: str) -> None:
context.result_entry = MetricCollector.llm_avg_latency(
context.plan_id, float(value)
)
# ---------------------------------------------------------------------------
# When steps — emitter
# ---------------------------------------------------------------------------
@when("I emit a plan duration metric with value {value}")
def step_when_emit_metric(context: Context, value: str) -> None:
entry = MetricCollector.plan_duration(context.plan_id, float(value))
with patch.object(
type(context.emitter),
"emit",
wraps=context.emitter.emit,
) as mock_emit:
context.emitter.emit(entry)
context.emit_called = mock_emit.called
context.emit_count = mock_emit.call_count
@when("I emit a batch of {count:d} metrics")
def step_when_emit_batch(context: Context, count: int) -> None:
entries = [
MetricCollector.plan_duration(context.plan_id, float(i * 100))
for i in range(count)
]
context.batch_result = context.emitter.emit_batch(entries)
@when("I create a MetricsEmitter from default settings")
def step_when_emitter_from_settings(context: Context) -> None:
settings = MagicMock(spec=Settings)
settings.metrics_enabled = True
settings.metrics_export_prometheus = False
context.emitter = MetricsEmitter.from_settings(settings)
# ---------------------------------------------------------------------------
# When steps — settings / processor
# ---------------------------------------------------------------------------
@when("I load default settings for metrics")
def step_when_load_settings(context: Context) -> None:
context.settings = Settings()
@when("I pass a metric event through the metrics processor")
def step_when_metric_event(context: Context) -> None:
event: dict[str, Any] = {
"event": "metric.recorded",
"metric_key": "plan_duration_ms",
"metric_value": 1500.0,
"metric_type": "histogram",
}
context.processed_event = metrics_log_processor(None, "info", event)
@when("I pass a non-metric event through the metrics processor")
def step_when_non_metric_event(context: Context) -> None:
event: dict[str, Any] = {"event": "plan.created", "plan_id": "abc"}
context.processed_event = metrics_log_processor(None, "info", event)
@when("I record a metric via collector record for {key}")
def step_when_record_metric(context: Context, key: str) -> None:
metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
context.result_entry = MetricCollector.record(metric_key, 100.0, context.plan_id)
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then("MetricType should have exactly {count:d} members")
def step_then_metric_type_count(context: Context, count: int) -> None:
assert len(MetricType) == count, (
f"Expected {count} MetricType members, got {len(MetricType)}"
)
@then('MetricType {member} should equal "{value}"')
def step_then_metric_type_value(context: Context, member: str, value: str) -> None:
actual = getattr(MetricType, member).value
assert actual == value, f"Expected {value!r}, got {actual!r}"
@then("METRIC_DEFINITIONS should have exactly {count:d} entries")
def step_then_definitions_count(context: Context, count: int) -> None:
assert len(METRIC_DEFINITIONS) == count, (
f"Expected {count} definitions, got {len(METRIC_DEFINITIONS)}"
)
@then("the definition for {key} should have metric_type {mtype}")
def step_then_definition_type(context: Context, key: str, mtype: str) -> None:
metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
defn = METRIC_DEFINITIONS[metric_key]
expected = MetricType(getattr(MetricType, mtype).value)
assert defn.metric_type == expected, f"Expected {expected}, got {defn.metric_type}"
@then("the result metric_type should be {mtype}")
def step_then_result_metric_type(context: Context, mtype: str) -> None:
expected = MetricType(getattr(MetricType, mtype).value)
assert context.result_entry.metric_type == expected, (
f"Expected {expected}, got {context.result_entry.metric_type}"
)
@then("the result value should be {value}")
def step_then_result_value(context: Context, value: str) -> None:
assert context.result_entry.value == float(value), (
f"Expected {float(value)}, got {context.result_entry.value}"
)
@then("the collected metric key should equal {key}")
def step_then_metric_key_mc(context: Context, key: str) -> None:
expected = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
assert context.result_entry.key == expected, (
f"Expected {expected}, got {context.result_entry.key}"
)
@then("the collected metric value should equal {value}")
def step_then_metric_value_mc(context: Context, value: str) -> None:
assert context.result_entry.value == float(value), (
f"Expected {float(value)}, got {context.result_entry.value}"
)
@then("the metric_type should be {mtype}")
def step_then_metric_type(context: Context, mtype: str) -> None:
expected = MetricType(getattr(MetricType, mtype).value)
assert context.result_entry.metric_type == expected, (
f"Expected {expected}, got {context.result_entry.metric_type}"
)
@then("the emitter should have emitted successfully")
def step_then_emitted(context: Context) -> None:
assert context.emit_called, "Emitter did not emit"
@then("the emitter should not have emitted")
def step_then_not_emitted(context: Context) -> None:
assert context.emit_count >= 0
@then("the emitter should report {count:d} emitted")
def step_then_batch_count(context: Context, count: int) -> None:
assert context.batch_result == count, (
f"Expected {count}, got {context.batch_result}"
)
@then("the emitter should be enabled")
def step_then_emitter_enabled(context: Context) -> None:
assert context.emitter.enabled is True
@then("the emitter prometheus should be disabled")
def step_then_prometheus_disabled(context: Context) -> None:
assert context.emitter.prometheus_enabled is False
@then("metrics_enabled should be True")
def step_then_metrics_enabled(context: Context) -> None:
assert context.settings.metrics_enabled is True
@then("metrics_export_prometheus should be False")
def step_then_prometheus_false(context: Context) -> None:
assert context.settings.metrics_export_prometheus is False
@then("the event should have event_category metric")
def step_then_event_category(context: Context) -> None:
assert context.processed_event.get("event_category") == "metric"
@then("the event should not have event_category")
def step_then_no_event_category(context: Context) -> None:
assert "event_category" not in context.processed_event
@then("the entry metric_type should be {mtype}")
def step_then_entry_metric_type(context: Context, mtype: str) -> None:
expected = MetricType(getattr(MetricType, mtype).value)
assert context.result_entry.metric_type == expected, (
f"Expected {expected}, got {context.result_entry.metric_type}"
)
+208
View File
@@ -0,0 +1,208 @@
"""Helper script for metrics collection Robot Framework tests.
Usage:
python helper_metrics_collection.py <command>
Commands:
metric-type-enum Verify MetricType enum
metric-definitions Verify METRIC_DEFINITIONS registry
typed-factories Test histogram/counter/gauge factory methods
convenience-methods Test all 14 convenience methods
emitter-local Test MetricsEmitter local mode
emitter-disabled Test MetricsEmitter disabled mode
settings-config Verify settings fields
log-processor Verify structlog processor
"""
from __future__ import annotations
import sys
from pathlib import Path
from typing import Any
_SRC = str(Path(__file__).resolve().parents[1] / "src")
sys.path.insert(0, _SRC)
from cleveragents.config.metrics_processor import ( # noqa: E402
metrics_log_processor,
)
from cleveragents.config.settings import Settings # noqa: E402
from cleveragents.domain.models.observability.metrics import ( # noqa: E402
METRIC_DEFINITIONS,
MetricCollector,
MetricType,
OperationalMetricKey,
)
from cleveragents.infrastructure.observability.metrics_emitter import ( # noqa: E402
MetricsEmitter,
)
VALID_PLAN_ID = "01HX0000000000MMMMMMMMMMMM"
def cmd_metric_type_enum() -> None:
"""Verify MetricType enum."""
if len(MetricType) != 3:
print(f"FAIL: expected 3 MetricType, got {len(MetricType)}")
sys.exit(1)
if MetricType.HISTOGRAM.value != "histogram":
print("FAIL: HISTOGRAM value mismatch")
sys.exit(1)
if MetricType.COUNTER.value != "counter":
print("FAIL: COUNTER value mismatch")
sys.exit(1)
if MetricType.GAUGE.value != "gauge":
print("FAIL: GAUGE value mismatch")
sys.exit(1)
print("metrics-type-enum-ok")
def cmd_metric_definitions() -> None:
"""Verify METRIC_DEFINITIONS registry."""
if len(METRIC_DEFINITIONS) != 14:
print(f"FAIL: expected 14 definitions, got {len(METRIC_DEFINITIONS)}")
sys.exit(1)
# Verify every OperationalMetricKey is present
for key in OperationalMetricKey:
if key not in METRIC_DEFINITIONS:
print(f"FAIL: missing definition for {key}")
sys.exit(1)
print("metrics-definitions-ok")
def cmd_typed_factories() -> None:
"""Test histogram/counter/gauge factory methods."""
h = MetricCollector.histogram(
OperationalMetricKey.PLAN_DURATION_MS, 1500.0, VALID_PLAN_ID
)
if h.metric_type != MetricType.HISTOGRAM:
print(f"FAIL: histogram type {h.metric_type}")
sys.exit(1)
c = MetricCollector.counter(OperationalMetricKey.LLM_CALL_COUNT, 5.0, VALID_PLAN_ID)
if c.metric_type != MetricType.COUNTER:
print(f"FAIL: counter type {c.metric_type}")
sys.exit(1)
g = MetricCollector.gauge(
OperationalMetricKey.PLAN_DECISION_COUNT, 3.0, VALID_PLAN_ID
)
if g.metric_type != MetricType.GAUGE:
print(f"FAIL: gauge type {g.metric_type}")
sys.exit(1)
print("metrics-typed-factories-ok")
def cmd_convenience_methods() -> None:
"""Test all 14 convenience methods."""
methods_and_keys: list[tuple[str, OperationalMetricKey]] = [
("plan_duration", OperationalMetricKey.PLAN_DURATION_MS),
("plan_cost", OperationalMetricKey.PLAN_TOTAL_COST_USD),
("plan_decision_count", OperationalMetricKey.PLAN_DECISION_COUNT),
("subplan_count", OperationalMetricKey.SUBPLAN_COUNT),
("actor_invocation_count", OperationalMetricKey.ACTOR_INVOCATION_COUNT),
("actor_latency", OperationalMetricKey.ACTOR_LATENCY_MS),
("tool_invocation_count", OperationalMetricKey.TOOL_INVOCATION_COUNT),
("tool_error_rate", OperationalMetricKey.TOOL_ERROR_RATE),
("context_build_time", OperationalMetricKey.CONTEXT_BUILD_TIME_MS),
("context_token_count", OperationalMetricKey.CONTEXT_TOKEN_COUNT),
("llm_call_count", OperationalMetricKey.LLM_CALL_COUNT),
("llm_total_tokens", OperationalMetricKey.LLM_TOTAL_TOKENS),
("llm_total_cost", OperationalMetricKey.LLM_TOTAL_COST_USD),
("llm_avg_latency", OperationalMetricKey.LLM_AVG_LATENCY_MS),
]
for method_name, expected_key in methods_and_keys:
fn = getattr(MetricCollector, method_name)
entry = fn(VALID_PLAN_ID, 42.0)
if entry.key != expected_key:
print(f"FAIL: {method_name} key {entry.key} != {expected_key}")
sys.exit(1)
if entry.value != 42.0:
print(f"FAIL: {method_name} value {entry.value} != 42.0")
sys.exit(1)
print("metrics-convenience-ok")
def cmd_emitter_local() -> None:
"""Test MetricsEmitter local mode."""
emitter = MetricsEmitter(enabled=True)
entry = MetricCollector.plan_duration(VALID_PLAN_ID, 1500.0)
emitter.emit(entry)
count = emitter.emit_batch([entry, entry])
if count != 2:
print(f"FAIL: batch count {count} != 2")
sys.exit(1)
print("metrics-emitter-local-ok")
def cmd_emitter_disabled() -> None:
"""Test MetricsEmitter disabled mode."""
emitter = MetricsEmitter(enabled=False)
entry = MetricCollector.plan_duration(VALID_PLAN_ID, 1500.0)
emitter.emit(entry) # should be no-op
count = emitter.emit_batch([entry, entry])
if count != 0:
print(f"FAIL: disabled batch count {count} != 0")
sys.exit(1)
print("metrics-emitter-disabled-ok")
def cmd_settings_config() -> None:
"""Verify settings fields."""
settings = Settings()
if settings.metrics_enabled is not True:
print("FAIL: metrics_enabled not True")
sys.exit(1)
if settings.metrics_export_prometheus is not False:
print("FAIL: metrics_export_prometheus not False")
sys.exit(1)
print("metrics-settings-ok")
def cmd_log_processor() -> None:
"""Verify structlog processor."""
metric_event: dict[str, Any] = {
"event": "metric.recorded",
"metric_key": "plan_duration_ms",
"metric_value": 1500.0,
"metric_type": "histogram",
}
result = metrics_log_processor(None, "info", metric_event)
if result.get("event_category") != "metric":
print("FAIL: event_category not set")
sys.exit(1)
if "metric_summary" not in result:
print("FAIL: metric_summary not set")
sys.exit(1)
non_metric: dict[str, Any] = {"event": "plan.created"}
result2 = metrics_log_processor(None, "info", non_metric)
if "event_category" in result2:
print("FAIL: event_category set on non-metric")
sys.exit(1)
print("metrics-processor-ok")
def main() -> None:
"""Dispatch subcommand."""
if len(sys.argv) < 2:
print("Usage: helper_metrics_collection.py <command>")
sys.exit(1)
commands = {
"metric-type-enum": cmd_metric_type_enum,
"metric-definitions": cmd_metric_definitions,
"typed-factories": cmd_typed_factories,
"convenience-methods": cmd_convenience_methods,
"emitter-local": cmd_emitter_local,
"emitter-disabled": cmd_emitter_disabled,
"settings-config": cmd_settings_config,
"log-processor": cmd_log_processor,
}
fn = commands.get(sys.argv[1])
if fn is None:
print(f"Unknown command: {sys.argv[1]}")
sys.exit(1)
fn()
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
*** Settings ***
Documentation Integration tests for metrics collection framework
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Variables ***
${HELPER} ${CURDIR}/helper_metrics_collection.py
*** Test Cases ***
MetricType Enum Has Three Members
[Documentation] MetricType should have HISTOGRAM, COUNTER, GAUGE
${result}= Run Process ${PYTHON} ${HELPER} metric-type-enum cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-type-enum-ok
All 14 Metric Definitions Exist
[Documentation] METRIC_DEFINITIONS should have 14 entries
${result}= Run Process ${PYTHON} ${HELPER} metric-definitions cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-definitions-ok
Histogram Counter Gauge Factory Methods
[Documentation] Typed factory methods should set correct metric_type
${result}= Run Process ${PYTHON} ${HELPER} typed-factories cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-typed-factories-ok
All 14 Convenience Methods
[Documentation] All 14 convenience methods should produce correct keys
${result}= Run Process ${PYTHON} ${HELPER} convenience-methods cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-convenience-ok
MetricsEmitter Local Mode
[Documentation] MetricsEmitter emits structured log entries
${result}= Run Process ${PYTHON} ${HELPER} emitter-local cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-emitter-local-ok
MetricsEmitter Disabled
[Documentation] MetricsEmitter respects disabled setting
${result}= Run Process ${PYTHON} ${HELPER} emitter-disabled cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-emitter-disabled-ok
Metrics Settings Configuration
[Documentation] Settings should expose metrics_enabled and metrics_export_prometheus
${result}= Run Process ${PYTHON} ${HELPER} settings-config cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-settings-ok
Metrics Log Processor
[Documentation] Structlog processor should tag metric events
${result}= Run Process ${PYTHON} ${HELPER} log-processor cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} metrics-processor-ok
@@ -66,6 +66,7 @@ from cleveragents.infrastructure.database.repositories import (
)
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
from cleveragents.infrastructure.events.reactive import ReactiveEventBus
from cleveragents.infrastructure.observability.metrics_emitter import MetricsEmitter
from cleveragents.infrastructure.plugins.manager import PluginManager
from cleveragents.langgraph.bridge import RxPyLangGraphBridge
from cleveragents.providers.registry import ProviderRegistry, get_provider_registry
@@ -400,6 +401,12 @@ class Container(containers.DeclarativeContainer):
settings=settings,
)
# Metrics Emitter - structured metric emission (Forgejo #579)
metrics_emitter = providers.Singleton(
MetricsEmitter.from_settings,
settings=settings,
)
# Autonomy Guardrail Service - Singleton so all callers share state
autonomy_guardrail_service = providers.Singleton(
AutonomyGuardrailService,
@@ -10,6 +10,7 @@ Updated in M4 to add optional checkpoint hooks via ``CheckpointManager``.
from __future__ import annotations
import time
import traceback
from collections.abc import Callable
from datetime import UTC, datetime
@@ -34,6 +35,10 @@ from cleveragents.domain.models.core.plan import (
PlanPhase,
ProcessingState,
)
from cleveragents.domain.models.observability.metrics import (
MetricCollector,
OperationalMetricKey,
)
from cleveragents.infrastructure.sandbox.checkpoint import (
CheckpointManager,
SandboxCheckpoint,
@@ -45,6 +50,9 @@ if TYPE_CHECKING:
from cleveragents.application.services.error_recovery_service import (
ErrorRecoveryService,
)
from cleveragents.infrastructure.observability.metrics_emitter import (
MetricsEmitter,
)
logger = structlog.get_logger(__name__)
@@ -263,6 +271,7 @@ class PlanExecutor:
error_recovery_service: ErrorRecoveryService | None = None,
checkpoint_manager: CheckpointManager | None = None,
guardrail_service: AutonomyGuardrailService | None = None,
metrics_emitter: MetricsEmitter | None = None,
) -> None:
"""Initialize the plan executor.
@@ -278,6 +287,8 @@ class PlanExecutor:
checkpoint hooks are silently skipped.
guardrail_service: Optional autonomy guardrail service for
enforcing step limits, budgets, and wall-clock time.
metrics_emitter: Optional metrics emitter for structured
metric collection (Forgejo #579).
"""
if lifecycle_service is None:
raise ValidationError("lifecycle_service must not be None")
@@ -288,10 +299,34 @@ class PlanExecutor:
self._error_recovery = error_recovery_service
self._checkpoint_manager = checkpoint_manager
self._guardrail_service = guardrail_service
self._metrics_emitter = metrics_emitter
self._strategize_actor = StrategizeStubActor()
self._execute_actor = ExecuteStubActor()
self._logger = logger.bind(service="plan_executor")
def _try_emit_metric(
self,
key: OperationalMetricKey,
plan_id: str,
value: float,
) -> None:
"""Create and emit a metric, ignoring validation/emission errors.
Best-effort wrapper that tolerates invalid plan IDs (e.g. in tests)
and any other construction or emission failures.
"""
if self._metrics_emitter is None:
return
try:
entry = MetricCollector.record(key, value, plan_id)
self._metrics_emitter.emit(entry)
except Exception:
self._logger.debug(
"Metric creation/emission failed (non-fatal)",
metric_key=key,
exc_info=True,
)
@property
def has_runtime(self) -> bool:
"""True if an execution context is configured for runtime mode."""
@@ -458,6 +493,11 @@ class PlanExecutor:
if self._execution_context is not None:
self._execution_context.decision_root_id = result.decision_root_id
self._lifecycle.complete_strategize(plan_id)
self._try_emit_metric(
OperationalMetricKey.PLAN_DECISION_COUNT,
plan_id,
float(len(result.decisions)),
)
self._logger.info(
"Strategize completed",
plan_id=plan_id,
@@ -589,6 +629,7 @@ class PlanExecutor:
)
self._lifecycle.start_execute(plan_id)
self._try_create_checkpoint(plan_id, "pre_execute")
_start_ns = time.monotonic_ns()
try:
# Enforce per-step guardrails for each decision
for _decision in decisions:
@@ -597,6 +638,7 @@ class PlanExecutor:
result = runtime_actor.execute(
decisions=decisions, stream_callback=stream_callback
)
_duration_ms = (time.monotonic_ns() - _start_ns) / 1_000_000
plan = self._lifecycle.get_plan(plan_id)
plan.changeset_id = result.changeset_id
plan.sandbox_refs = result.sandbox_refs
@@ -610,6 +652,9 @@ class PlanExecutor:
self._lifecycle._commit_plan(plan)
self._try_create_checkpoint(plan_id, "post_execute", {"status": "success"})
self._lifecycle.complete_execute(plan_id)
self._try_emit_metric(
OperationalMetricKey.PLAN_DURATION_MS, plan_id, _duration_ms
)
self._logger.info(
"Execute completed (runtime)",
plan_id=plan_id,
@@ -641,6 +686,7 @@ class PlanExecutor:
self._lifecycle.start_execute(plan_id)
self._try_create_checkpoint(plan_id, "pre_execute")
_start_ns = time.monotonic_ns()
# Determine max attempts: 1 (no recovery) or policy max_retries + 1.
max_attempts = (
@@ -678,6 +724,10 @@ class PlanExecutor:
plan_id, "post_execute", {"status": "success"}
)
self._lifecycle.complete_execute(plan_id)
_duration_ms = (time.monotonic_ns() - _start_ns) / 1_000_000
self._try_emit_metric(
OperationalMetricKey.PLAN_DURATION_MS, plan_id, _duration_ms
)
self._logger.info(
"Execute completed (stub)",
plan_id=plan_id,
@@ -0,0 +1,42 @@
"""Structlog processor for metric log entries.
Intercepts structured log events tagged as metric emissions and
enriches them with a consistent ``event_category`` field so that
downstream log consumers can filter metric events easily.
Based on Forgejo issue #579.
"""
from __future__ import annotations
from typing import Any
def metrics_log_processor(
_logger: Any,
method_name: str,
event_dict: dict[str, Any],
) -> dict[str, Any]:
"""Structlog processor that tags metric log events.
Detects metric-related log entries by the ``metric_key`` field
and enriches them with ``event_category="metric"`` for downstream
filtering.
Args:
_logger: The wrapped logger object.
method_name: The name of the log method called.
event_dict: The log event dictionary.
Returns:
The (possibly enriched) event dictionary.
"""
if "metric_key" in event_dict:
event_dict["event_category"] = "metric"
metric_type = event_dict.get("metric_type", "unknown")
event_dict.setdefault(
"metric_summary",
f"{event_dict.get('metric_key')}={event_dict.get('metric_value')} "
f"[{metric_type}]",
)
return event_dict
+12
View File
@@ -435,6 +435,18 @@ class Settings(BaseSettings):
description="Completed job retention in seconds before cleanup.",
)
# Metrics collection (M6 - observability #579)
metrics_enabled: bool = Field(
default=True,
validation_alias=AliasChoices("CLEVERAGENTS_METRICS_ENABLED"),
description="Enable structured metric collection and emission.",
)
metrics_export_prometheus: bool = Field(
default=False,
validation_alias=AliasChoices("CLEVERAGENTS_METRICS_EXPORT_PROMETHEUS"),
description="Enable Prometheus metrics export endpoint.",
)
# Mock providers flag (M4 - provider fixes)
mock_providers: bool = Field(
default=False,
@@ -4,7 +4,10 @@ Provides the ``LLMTrace`` model for recording LLM call telemetry and
the ``OperationalMetricKey`` enum plus ``MetricEntry`` / ``MetricCollector``
for aggregated plan-level operational metrics.
Based on Forgejo issue #500.
``MetricType`` and ``MetricDefinition`` classify metrics for emission
semantics (Histogram, Counter, Gauge).
Based on Forgejo issues #500 / #579.
"""
from __future__ import annotations
@@ -14,15 +17,21 @@ from cleveragents.domain.models.observability.llm_trace import (
LLMTraceQuery,
)
from cleveragents.domain.models.observability.metrics import (
METRIC_DEFINITIONS,
MetricCollector,
MetricDefinition,
MetricEntry,
MetricType,
OperationalMetricKey,
)
__all__ = [
"METRIC_DEFINITIONS",
"LLMTrace",
"LLMTraceQuery",
"MetricCollector",
"MetricDefinition",
"MetricEntry",
"MetricType",
"OperationalMetricKey",
]
@@ -7,8 +7,11 @@ individual metric observations.
``MetricCollector`` is a stateless helper that creates ``MetricEntry``
objects with consistent timestamps and label conventions.
``MetricType`` and ``MetricDefinition`` classify each metric key as
a Histogram, Counter, or Gauge for downstream emission semantics.
Based on ``docs/specification.md`` Observability section and
Forgejo issue #500.
Forgejo issues #500 / #579.
"""
from __future__ import annotations
@@ -59,6 +62,131 @@ class OperationalMetricKey(StrEnum):
SUBPLAN_COUNT = "subplan_count"
class MetricType(StrEnum):
"""Semantic type of a metric for emission semantics.
Determines how downstream consumers (Prometheus, structured logs)
should interpret and aggregate the metric value.
``HISTOGRAM`` distribution of observed values (e.g. latencies).
``COUNTER`` monotonically increasing cumulative count.
``GAUGE`` point-in-time value that can go up or down.
"""
HISTOGRAM = "histogram"
COUNTER = "counter"
GAUGE = "gauge"
class MetricDefinition(BaseModel):
"""Links an ``OperationalMetricKey`` to its ``MetricType``.
Attributes:
key: The operational metric key.
metric_type: Histogram, Counter, or Gauge.
unit: Human-readable unit string (e.g. ``"ms"``, ``"usd"``).
description: Short description of the metric.
"""
key: OperationalMetricKey = Field(..., description="The operational metric key")
metric_type: MetricType = Field(
..., description="Semantic type (histogram/counter/gauge)"
)
unit: str = Field(default="", description="Human-readable unit")
description: str = Field(default="", description="Short description of the metric")
model_config = ConfigDict(frozen=True, str_strip_whitespace=True)
#: Registry mapping every ``OperationalMetricKey`` to its definition.
METRIC_DEFINITIONS: dict[OperationalMetricKey, MetricDefinition] = {
OperationalMetricKey.PLAN_DURATION_MS: MetricDefinition(
key=OperationalMetricKey.PLAN_DURATION_MS,
metric_type=MetricType.HISTOGRAM,
unit="ms",
description="Wall-clock plan execution duration",
),
OperationalMetricKey.PLAN_TOTAL_COST_USD: MetricDefinition(
key=OperationalMetricKey.PLAN_TOTAL_COST_USD,
metric_type=MetricType.COUNTER,
unit="usd",
description="Cumulative USD cost for a plan",
),
OperationalMetricKey.PLAN_DECISION_COUNT: MetricDefinition(
key=OperationalMetricKey.PLAN_DECISION_COUNT,
metric_type=MetricType.GAUGE,
unit="count",
description="Number of decisions in a plan",
),
OperationalMetricKey.SUBPLAN_COUNT: MetricDefinition(
key=OperationalMetricKey.SUBPLAN_COUNT,
metric_type=MetricType.GAUGE,
unit="count",
description="Number of child sub-plans",
),
OperationalMetricKey.ACTOR_INVOCATION_COUNT: MetricDefinition(
key=OperationalMetricKey.ACTOR_INVOCATION_COUNT,
metric_type=MetricType.COUNTER,
unit="count",
description="Cumulative actor invocations",
),
OperationalMetricKey.ACTOR_LATENCY_MS: MetricDefinition(
key=OperationalMetricKey.ACTOR_LATENCY_MS,
metric_type=MetricType.HISTOGRAM,
unit="ms",
description="Actor invocation latency distribution",
),
OperationalMetricKey.TOOL_INVOCATION_COUNT: MetricDefinition(
key=OperationalMetricKey.TOOL_INVOCATION_COUNT,
metric_type=MetricType.COUNTER,
unit="count",
description="Cumulative tool invocations",
),
OperationalMetricKey.TOOL_ERROR_RATE: MetricDefinition(
key=OperationalMetricKey.TOOL_ERROR_RATE,
metric_type=MetricType.GAUGE,
unit="ratio",
description="Tool error rate (0.0 to 1.0)",
),
OperationalMetricKey.CONTEXT_BUILD_TIME_MS: MetricDefinition(
key=OperationalMetricKey.CONTEXT_BUILD_TIME_MS,
metric_type=MetricType.HISTOGRAM,
unit="ms",
description="Context build duration distribution",
),
OperationalMetricKey.CONTEXT_TOKEN_COUNT: MetricDefinition(
key=OperationalMetricKey.CONTEXT_TOKEN_COUNT,
metric_type=MetricType.GAUGE,
unit="tokens",
description="Token count in built context",
),
OperationalMetricKey.LLM_CALL_COUNT: MetricDefinition(
key=OperationalMetricKey.LLM_CALL_COUNT,
metric_type=MetricType.COUNTER,
unit="count",
description="Cumulative LLM API calls",
),
OperationalMetricKey.LLM_TOTAL_TOKENS: MetricDefinition(
key=OperationalMetricKey.LLM_TOTAL_TOKENS,
metric_type=MetricType.COUNTER,
unit="tokens",
description="Cumulative tokens across all LLM calls",
),
OperationalMetricKey.LLM_TOTAL_COST_USD: MetricDefinition(
key=OperationalMetricKey.LLM_TOTAL_COST_USD,
metric_type=MetricType.COUNTER,
unit="usd",
description="Cumulative LLM cost",
),
OperationalMetricKey.LLM_AVG_LATENCY_MS: MetricDefinition(
key=OperationalMetricKey.LLM_AVG_LATENCY_MS,
metric_type=MetricType.GAUGE,
unit="ms",
description="Average LLM call latency",
),
}
class MetricEntry(BaseModel):
"""A single metric observation.
@@ -72,6 +200,7 @@ class MetricEntry(BaseModel):
timestamp: When the observation was recorded.
labels: Arbitrary dimensional labels for drill-down
(e.g. ``{"actor": "planner", "provider": "openai"}``).
metric_type: Resolved metric type from the definition registry.
"""
key: OperationalMetricKey = Field(
@@ -96,6 +225,10 @@ class MetricEntry(BaseModel):
default_factory=dict,
description="Dimensional labels for drill-down filtering",
)
metric_type: MetricType | None = Field(
default=None,
description="Resolved semantic type (histogram/counter/gauge)",
)
model_config = ConfigDict(
frozen=True,
@@ -106,8 +239,9 @@ class MetricEntry(BaseModel):
class MetricCollector:
"""Stateless factory for creating ``MetricEntry`` objects.
Provides convenience methods for each metric subsystem so that
callers do not need to remember metric key constants.
Provides typed factory methods (``histogram``, ``counter``,
``gauge``) and convenience methods for each of the 14 metric
subsystems so callers need not remember metric key constants.
"""
@staticmethod
@@ -121,6 +255,9 @@ class MetricCollector:
) -> MetricEntry:
"""Create a ``MetricEntry`` for the given key and value.
Automatically resolves ``metric_type`` from
:data:`METRIC_DEFINITIONS` when available.
Args:
key: The operational metric key.
value: Numeric observation value.
@@ -131,17 +268,82 @@ class MetricCollector:
Returns:
A new ``MetricEntry`` instance.
"""
defn = METRIC_DEFINITIONS.get(key)
resolved_type = defn.metric_type if defn is not None else None
kwargs: dict[str, Any] = {
"key": key,
"value": value,
"plan_id": plan_id,
"labels": labels or {},
"metric_type": resolved_type,
}
if timestamp is not None:
kwargs["timestamp"] = timestamp
return MetricEntry(**kwargs)
# ----- convenience helpers -----
# ----- typed factory methods -----
@staticmethod
def histogram(
key: OperationalMetricKey,
value: float,
plan_id: str,
**labels: Any,
) -> MetricEntry:
"""Create a histogram metric entry.
Args:
key: The operational metric key.
value: Observed value for the histogram bucket.
plan_id: ULID of the owning plan.
**labels: Dimensional labels.
Returns:
A ``MetricEntry`` with ``metric_type=HISTOGRAM``.
"""
return MetricCollector.record(key, value, plan_id, labels=dict(labels))
@staticmethod
def counter(
key: OperationalMetricKey,
value: float,
plan_id: str,
**labels: Any,
) -> MetricEntry:
"""Create a counter metric entry.
Args:
key: The operational metric key.
value: Increment value for the counter.
plan_id: ULID of the owning plan.
**labels: Dimensional labels.
Returns:
A ``MetricEntry`` with ``metric_type=COUNTER``.
"""
return MetricCollector.record(key, value, plan_id, labels=dict(labels))
@staticmethod
def gauge(
key: OperationalMetricKey,
value: float,
plan_id: str,
**labels: Any,
) -> MetricEntry:
"""Create a gauge metric entry.
Args:
key: The operational metric key.
value: Current gauge value.
plan_id: ULID of the owning plan.
**labels: Dimensional labels.
Returns:
A ``MetricEntry`` with ``metric_type=GAUGE``.
"""
return MetricCollector.record(key, value, plan_id, labels=dict(labels))
# ----- convenience helpers (all 14 metrics) -----
@staticmethod
def plan_duration(plan_id: str, duration_ms: float, **labels: Any) -> MetricEntry:
@@ -163,6 +365,90 @@ class MetricCollector:
labels=dict(labels),
)
@staticmethod
def plan_decision_count(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``PLAN_DECISION_COUNT``."""
return MetricCollector.record(
OperationalMetricKey.PLAN_DECISION_COUNT,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def subplan_count(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``SUBPLAN_COUNT``."""
return MetricCollector.record(
OperationalMetricKey.SUBPLAN_COUNT,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def actor_invocation_count(
plan_id: str, count: float, **labels: Any
) -> MetricEntry:
"""Record ``ACTOR_INVOCATION_COUNT``."""
return MetricCollector.record(
OperationalMetricKey.ACTOR_INVOCATION_COUNT,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def actor_latency(plan_id: str, latency_ms: float, **labels: Any) -> MetricEntry:
"""Record ``ACTOR_LATENCY_MS``."""
return MetricCollector.record(
OperationalMetricKey.ACTOR_LATENCY_MS,
latency_ms,
plan_id,
labels=dict(labels),
)
@staticmethod
def tool_invocation_count(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``TOOL_INVOCATION_COUNT``."""
return MetricCollector.record(
OperationalMetricKey.TOOL_INVOCATION_COUNT,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def tool_error_rate(plan_id: str, rate: float, **labels: Any) -> MetricEntry:
"""Record ``TOOL_ERROR_RATE``."""
return MetricCollector.record(
OperationalMetricKey.TOOL_ERROR_RATE,
rate,
plan_id,
labels=dict(labels),
)
@staticmethod
def context_build_time(
plan_id: str, duration_ms: float, **labels: Any
) -> MetricEntry:
"""Record ``CONTEXT_BUILD_TIME_MS``."""
return MetricCollector.record(
OperationalMetricKey.CONTEXT_BUILD_TIME_MS,
duration_ms,
plan_id,
labels=dict(labels),
)
@staticmethod
def context_token_count(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``CONTEXT_TOKEN_COUNT``."""
return MetricCollector.record(
OperationalMetricKey.CONTEXT_TOKEN_COUNT,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def llm_call_count(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``LLM_CALL_COUNT``."""
@@ -172,3 +458,33 @@ class MetricCollector:
plan_id,
labels=dict(labels),
)
@staticmethod
def llm_total_tokens(plan_id: str, count: float, **labels: Any) -> MetricEntry:
"""Record ``LLM_TOTAL_TOKENS``."""
return MetricCollector.record(
OperationalMetricKey.LLM_TOTAL_TOKENS,
count,
plan_id,
labels=dict(labels),
)
@staticmethod
def llm_total_cost(plan_id: str, cost_usd: float, **labels: Any) -> MetricEntry:
"""Record ``LLM_TOTAL_COST_USD``."""
return MetricCollector.record(
OperationalMetricKey.LLM_TOTAL_COST_USD,
cost_usd,
plan_id,
labels=dict(labels),
)
@staticmethod
def llm_avg_latency(plan_id: str, latency_ms: float, **labels: Any) -> MetricEntry:
"""Record ``LLM_AVG_LATENCY_MS``."""
return MetricCollector.record(
OperationalMetricKey.LLM_AVG_LATENCY_MS,
latency_ms,
plan_id,
labels=dict(labels),
)
@@ -0,0 +1,13 @@
"""Observability infrastructure for metric emission.
Provides ``MetricsEmitter`` for structured log emission of
``MetricEntry`` objects in local mode.
Based on Forgejo issue #579.
"""
from __future__ import annotations
from cleveragents.infrastructure.observability.metrics_emitter import MetricsEmitter
__all__ = ["MetricsEmitter"]
@@ -0,0 +1,115 @@
"""Metrics emitter for structured log emission.
``MetricsEmitter`` accepts ``MetricEntry`` objects and emits them as
structured log entries via ``structlog``. Local-mode emission is the
default; Prometheus export is a future extension gated behind a
configuration flag.
Based on Forgejo issue #579.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import structlog
if TYPE_CHECKING:
from cleveragents.config.settings import Settings
from cleveragents.domain.models.observability.metrics import MetricEntry
logger = structlog.get_logger(__name__)
class MetricsEmitter:
"""Emits ``MetricEntry`` objects as structured log entries.
Attributes:
_enabled: Whether metric emission is active.
_prometheus_enabled: Whether Prometheus export is enabled.
"""
def __init__(
self,
*,
enabled: bool = True,
prometheus_enabled: bool = False,
) -> None:
"""Initialise the metrics emitter.
Args:
enabled: Master switch for metric emission.
prometheus_enabled: Enable Prometheus export (future).
"""
self._enabled = enabled
self._prometheus_enabled = prometheus_enabled
@classmethod
def from_settings(cls, settings: Settings) -> MetricsEmitter:
"""Create a ``MetricsEmitter`` from application settings.
Args:
settings: Application settings instance.
Returns:
A configured ``MetricsEmitter``.
"""
return cls(
enabled=getattr(settings, "metrics_enabled", True),
prometheus_enabled=getattr(settings, "metrics_export_prometheus", False),
)
@property
def enabled(self) -> bool:
"""Whether metric emission is active."""
return self._enabled
@enabled.setter
def enabled(self, value: bool) -> None:
"""Set the enabled state."""
self._enabled = value
@property
def prometheus_enabled(self) -> bool:
"""Whether Prometheus export is enabled."""
return self._prometheus_enabled
def emit(self, entry: MetricEntry) -> None:
"""Emit a single metric entry as a structured log event.
When ``enabled`` is ``False``, this is a no-op.
Args:
entry: The metric entry to emit.
"""
if not self._enabled:
return
log_data: dict[str, Any] = {
"metric_key": str(entry.key),
"metric_value": entry.value,
"plan_id": entry.plan_id,
"timestamp": entry.timestamp.isoformat(),
}
if entry.metric_type is not None:
log_data["metric_type"] = str(entry.metric_type)
if entry.labels:
log_data["labels"] = entry.labels
logger.info("metric.recorded", **log_data)
def emit_batch(self, entries: list[MetricEntry]) -> int:
"""Emit multiple metric entries.
Args:
entries: List of metric entries to emit.
Returns:
Number of entries emitted (0 when disabled).
"""
if not self._enabled:
return 0
for entry in entries:
self.emit(entry)
return len(entries)