From 8c81f1375886607c278cdbdb48b711974964cbbd Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Thu, 2 Apr 2026 23:59:37 +0000 Subject: [PATCH] fix(infra): resolve TLS handshake failure on git.dev.cleveragents.com The TLS handshake failure on git.dev.cleveragents.com was caused by the hostname being absent from the certificate's Subject Alternative Names (SANs), or by SNI virtual-host misconfiguration on the server side. This commit delivers the repository-side remediation: - scripts/check-tls-cert.py: New TLS certificate health-check script. Connects to a hostname, verifies the certificate's SANs include the target hostname, checks expiry, and reports errors/warnings. Accepts an injectable SSLContext for unit testing without real network access. Supports wildcard SAN matching and configurable expiry warning threshold. - docs/development/ops-runbook.md: New ops runbook documenting the full certificate renewal procedure (Let's Encrypt/certbot and manual CA), SNI misconfiguration diagnosis steps, expiry monitoring with cron, and recommended alert thresholds (30/14/7/0 days). - features/tls_certificate_check.feature: 14 Behave scenarios tagged @tdd_issue @tdd_issue_1543 covering: missing SAN detection, valid SAN acceptance, expired certificate detection, expiry warning threshold, TLS handshake errors, connection timeouts, connection refused, wildcard SAN matching, and _hostname_matches_san unit tests. - features/steps/tls_certificate_check_steps.py: Step definitions for the above feature, using unittest.mock to inject SSL contexts and socket connections so no real network calls are made. - mkdocs.yml: Added Ops Runbook to the Development section navigation. The actual server-side certificate renewal (adding git.dev.cleveragents.com as a SAN and reloading the web server) must be performed by the server administrator following the procedure in docs/development/ops-runbook.md. Closes #1543 ISSUES CLOSED: #1543 --- docs/development/ops-runbook.md | 274 ++++++++++++++++ features/steps/tls_certificate_check_steps.py | 282 ++++++++++++++++ features/tls_certificate_check.feature | 110 +++++++ mkdocs.yml | 1 + scripts/check-tls-cert.py | 308 ++++++++++++++++++ 5 files changed, 975 insertions(+) create mode 100644 docs/development/ops-runbook.md create mode 100644 features/steps/tls_certificate_check_steps.py create mode 100644 features/tls_certificate_check.feature create mode 100644 scripts/check-tls-cert.py diff --git a/docs/development/ops-runbook.md b/docs/development/ops-runbook.md new file mode 100644 index 000000000..9494fde05 --- /dev/null +++ b/docs/development/ops-runbook.md @@ -0,0 +1,274 @@ +# Ops Runbook: TLS Certificate Management + +This runbook documents procedures for diagnosing, renewing, and monitoring +TLS certificates for CleverAgents infrastructure hostnames. + +## Affected Hostnames + +| Hostname | Purpose | +|---|---| +| `git.cleverthis.com` | Primary Forgejo git server (production) | +| `git.dev.cleveragents.com` | Development Forgejo git server | +| `git.cleveragents.com` | Alias / secondary git endpoint | + +## Diagnosing TLS Issues + +### Symptom: `gnutls_handshake() failed: The server name sent was not recognized` + +This error indicates one of: + +1. **Missing SAN** — The TLS certificate does not list the target hostname as a + Subject Alternative Name (SAN). +2. **SNI misconfiguration** — The web server (nginx/caddy/traefik) is not routing + the SNI hostname to the correct virtual host. +3. **Expired certificate** — The certificate has passed its `notAfter` date. + +### Step 1: Inspect the Certificate + +Use `openssl s_client` to retrieve and inspect the certificate: + +```bash +# Check SANs and expiry for git.dev.cleveragents.com +openssl s_client \ + -connect git.dev.cleveragents.com:443 \ + -servername git.dev.cleveragents.com \ + 2>/dev/null | openssl x509 -noout -text | grep -A 10 "Subject Alternative Name" + +# Check expiry date +openssl s_client \ + -connect git.dev.cleveragents.com:443 \ + -servername git.dev.cleveragents.com \ + 2>/dev/null | openssl x509 -noout -dates +``` + +Or use the project's TLS check script (no external tools required): + +```bash +python scripts/check-tls-cert.py git.dev.cleveragents.com git.cleveragents.com +``` + +### Step 2: Identify the Root Cause + +| Symptom | Root Cause | Fix | +|---|---|---| +| `CERTIFICATE_VERIFY_FAILED` with `hostname mismatch` | Hostname not in SANs | Reissue certificate with correct SANs | +| `SSL_ERROR_RX_RECORD_TOO_LONG` | HTTP served on HTTPS port | Fix web server config | +| `gnutls_handshake() failed: The server name sent was not recognized` | SNI virtual host not configured | Add SNI vhost binding | +| Certificate expired | `notAfter` in the past | Renew certificate immediately | +| `GIT_SSL_NO_VERIFY` bypasses but still fails | Server-side SNI rejection | Fix SNI routing (not a client cert issue) | + +### Step 3: Verify SNI Routing + +If the certificate has the correct SANs but the error persists, the web server +may not be routing the SNI hostname to the correct virtual host: + +```bash +# nginx: check for server_name directive +grep -r "server_name.*git.dev.cleveragents.com" /etc/nginx/ + +# caddy: check Caddyfile +grep -r "git.dev.cleveragents.com" /etc/caddy/ + +# traefik: check router rules +grep -r "git.dev.cleveragents.com" /etc/traefik/ +``` + +--- + +## Certificate Renewal Procedure + +### Let's Encrypt (Certbot) — Recommended + +Most CleverAgents infrastructure uses Let's Encrypt certificates managed by +certbot. The renewal process is: + +#### 1. Verify certbot is installed and configured + +```bash +certbot --version +certbot certificates +``` + +#### 2. Check which certificate covers the affected hostname + +```bash +certbot certificates | grep -A 5 "git.dev.cleveragents.com" +``` + +#### 3. Renew the certificate + +```bash +# Dry run first to verify renewal will succeed +certbot renew --dry-run --cert-name + +# Actual renewal +certbot renew --cert-name +``` + +#### 4. If the hostname is missing from the certificate's SANs + +The certificate must be reissued with the additional hostname: + +```bash +# Add git.dev.cleveragents.com to an existing certificate +certbot certonly \ + --expand \ + --cert-name \ + -d git.cleverthis.com \ + -d git.cleveragents.com \ + -d git.dev.cleveragents.com +``` + +> **Note:** Replace `` with the actual certificate name shown by +> `certbot certificates`. The `-d` flags must list ALL hostnames the certificate +> should cover, including existing ones. + +#### 5. Reload the web server + +After renewal, reload the web server to pick up the new certificate: + +```bash +# nginx +systemctl reload nginx + +# caddy +systemctl reload caddy + +# traefik (usually auto-reloads; check logs) +journalctl -u traefik -n 50 +``` + +#### 6. Verify the fix + +```bash +python scripts/check-tls-cert.py git.dev.cleveragents.com git.cleveragents.com + +# Or with openssl +openssl s_client \ + -connect git.dev.cleveragents.com:443 \ + -servername git.dev.cleveragents.com \ + 2>/dev/null | openssl x509 -noout -text | grep -A 10 "Subject Alternative Name" +``` + +#### 7. Verify git clone works + +```bash +git clone https://git.dev.cleveragents.com/cleveragents/cleveragents-core.git /tmp/test-clone +rm -rf /tmp/test-clone +``` + +--- + +### Manual Certificate (non-Let's Encrypt) + +If the infrastructure uses manually managed certificates: + +1. Generate a new CSR including all required SANs: + + ```bash + # Create OpenSSL config with SANs + cat > /tmp/san.cnf << EOF + [req] + distinguished_name = req_distinguished_name + req_extensions = v3_req + prompt = no + + [req_distinguished_name] + CN = git.cleverthis.com + + [v3_req] + subjectAltName = @alt_names + + [alt_names] + DNS.1 = git.cleverthis.com + DNS.2 = git.cleveragents.com + DNS.3 = git.dev.cleveragents.com + EOF + + # Generate private key and CSR + openssl req -new -newkey rsa:4096 -nodes \ + -keyout /etc/ssl/private/git-cleverthis.key \ + -out /tmp/git-cleverthis.csr \ + -config /tmp/san.cnf + ``` + +2. Submit the CSR to your CA and obtain the signed certificate. + +3. Install the certificate and reload the web server. + +--- + +## Certificate Expiry Monitoring + +### Automated Monitoring with the TLS Check Script + +Run the TLS check script regularly to detect expiring certificates before they +cause outages: + +```bash +# Check all infrastructure hostnames with 30-day warning threshold +python scripts/check-tls-cert.py \ + git.cleverthis.com \ + git.cleveragents.com \ + git.dev.cleveragents.com \ + --warn-days 30 +``` + +### Recommended Alert Thresholds + +| Threshold | Action | +|---|---| +| 30 days | Warning — schedule renewal | +| 14 days | Urgent — renew immediately | +| 7 days | Critical — escalate to on-call | +| 0 days | Outage — emergency renewal | + +### Cron Job Setup + +Add a daily cron job to check certificate health: + +```bash +# /etc/cron.d/tls-cert-check +0 8 * * * root cd /path/to/cleveragents-core && \ + python scripts/check-tls-cert.py \ + git.cleverthis.com \ + git.cleveragents.com \ + git.dev.cleveragents.com \ + --warn-days 30 \ + | mail -s "TLS Cert Check" ops@cleverthis.com +``` + +### Let's Encrypt Auto-Renewal + +Let's Encrypt certificates are valid for 90 days. Certbot installs a systemd +timer or cron job that attempts renewal when the certificate has fewer than +30 days remaining: + +```bash +# Check certbot timer status +systemctl status certbot.timer + +# Manually trigger renewal check +certbot renew --dry-run +``` + +--- + +## Related Issues + +- **#1532** — BUG-HUNT: TLS Configuration Error on `git.cleveragents.com` +- **#1541** — TEST-INFRA: Unable to push to repository +- **#1543** — fix(infra): resolve TLS handshake failure on `git.dev.cleveragents.com` + +--- + +## Escalation Path + +1. **First responder:** Check certificate with `scripts/check-tls-cert.py` +2. **If renewal needed:** Follow [Certificate Renewal Procedure](#certificate-renewal-procedure) +3. **If SNI misconfiguration:** Contact server admin with web server config access +4. **If CA/PKI issue:** Escalate to infrastructure team lead + +--- + +*Last updated: 2026-04-02 — Added as part of issue #1543 resolution.* diff --git a/features/steps/tls_certificate_check_steps.py b/features/steps/tls_certificate_check_steps.py new file mode 100644 index 000000000..f7a423507 --- /dev/null +++ b/features/steps/tls_certificate_check_steps.py @@ -0,0 +1,282 @@ +"""Step definitions for tls_certificate_check.feature. + +Tests the ``scripts/check-tls-cert.py`` TLS certificate health-check script +using injected SSL contexts and mock socket connections so no real network +connections are made. All scenarios are tagged ``@tdd_issue @tdd_issue_1543`` +as permanent regression guards for the TLS handshake failure on +``git.dev.cleveragents.com`` (issue #1543). +""" + +from __future__ import annotations + +import datetime +import importlib.util as _ilu +import ssl +import sys +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context + +# --------------------------------------------------------------------------- +# Import the module under test +# --------------------------------------------------------------------------- + +_SCRIPTS_DIR = Path(__file__).parent.parent.parent / "scripts" + +# Import using importlib to handle the hyphenated filename. +# We must set __name__ and __package__ on the spec so that dataclasses +# can resolve the module's namespace correctly. +_spec = _ilu.spec_from_file_location( + "check_tls_cert", + _SCRIPTS_DIR / "check-tls-cert.py", +) +assert _spec is not None and _spec.loader is not None +_mod = _ilu.module_from_spec(_spec) +# Register the module in sys.modules BEFORE exec so that dataclasses +# (and other introspection tools) can look it up by __module__ name. +sys.modules["check_tls_cert"] = _mod +_spec.loader.exec_module(_mod) # type: ignore[union-attr] + +check_tls_certificate = _mod.check_tls_certificate +_hostname_matches_san = _mod._hostname_matches_san +CertCheckResult = _mod.CertCheckResult + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_cert( + cn: str, + sans: list[str], + not_after: datetime.datetime, +) -> dict: + """Build a minimal certificate dict in the format returned by ssl.getpeercert().""" + san_tuples = tuple(("DNS", san) for san in sans) + not_after_str = not_after.strftime("%b %d %H:%M:%S %Y GMT") + return { + "subject": ((("commonName", cn),),), + "subjectAltName": san_tuples, + "notAfter": not_after_str, + } + + +def _make_mock_ssl_context(cert: dict) -> MagicMock: + """Return a mock SSLContext whose wrap_socket yields a socket with *cert*.""" + mock_tls_sock = MagicMock() + mock_tls_sock.getpeercert.return_value = cert + mock_tls_sock.__enter__ = MagicMock(return_value=mock_tls_sock) + mock_tls_sock.__exit__ = MagicMock(return_value=False) + + mock_ctx = MagicMock(spec=ssl.SSLContext) + mock_ctx.wrap_socket.return_value = mock_tls_sock + return mock_ctx + + +def _make_mock_raw_sock() -> MagicMock: + """Return a mock raw socket for use as a context manager.""" + mock_raw = MagicMock() + mock_raw.__enter__ = MagicMock(return_value=mock_raw) + mock_raw.__exit__ = MagicMock(return_value=False) + return mock_raw + + +# --------------------------------------------------------------------------- +# Given — certificate setup +# --------------------------------------------------------------------------- + + +@given('a TLS certificate for "{cn}" with SANs "{sans_str}"') +def step_cert_with_sans(context: Context, cn: str, sans_str: str) -> None: + """Set up a mock certificate with the given CN and SANs.""" + context.cert_cn = cn + context.cert_sans = [s.strip() for s in sans_str.split(",")] + # Default expiry: 90 days from now (overridden by subsequent Given steps) + context.cert_not_after = datetime.datetime.now( + tz=datetime.UTC + ) + datetime.timedelta(days=90) + context.ssl_error = None + context.timeout_error = False + context.connection_error = None + + +@given("the certificate expires in {days:d} days") +def step_cert_expires_in(context: Context, days: int) -> None: + """Set the certificate expiry to *days* days from now.""" + context.cert_not_after = datetime.datetime.now( + tz=datetime.UTC + ) + datetime.timedelta(days=days) + + +@given("the certificate expired {days:d} days ago") +def step_cert_expired_ago(context: Context, days: int) -> None: + """Set the certificate expiry to *days* days in the past.""" + context.cert_not_after = datetime.datetime.now( + tz=datetime.UTC + ) - datetime.timedelta(days=days) + + +@given('the TLS connection raises an SSLError "{reason}"') +def step_ssl_error(context: Context, reason: str) -> None: + """Configure the mock to raise an SSLCertVerificationError.""" + context.ssl_error = ssl.SSLCertVerificationError(reason) + context.connection_error = None + context.timeout_error = False + + +@given("the TLS connection times out") +def step_timeout_error(context: Context) -> None: + """Configure the mock to raise a TimeoutError.""" + context.timeout_error = True + context.ssl_error = None + context.connection_error = None + + +@given("the TLS connection is refused") +def step_connection_refused(context: Context) -> None: + """Configure the mock to raise a ConnectionRefusedError.""" + context.connection_error = ConnectionRefusedError("Connection refused") + context.ssl_error = None + context.timeout_error = False + + +# --------------------------------------------------------------------------- +# When — run the check +# --------------------------------------------------------------------------- + + +def _run_check(context: Context, hostname: str, warn_days: int = 30) -> None: + """Execute check_tls_certificate with appropriate mocks.""" + ssl_error = getattr(context, "ssl_error", None) + timeout_error = getattr(context, "timeout_error", False) + connection_error = getattr(context, "connection_error", None) + + if ssl_error is not None: + # Simulate SSLCertVerificationError during wrap_socket + mock_ctx = MagicMock(spec=ssl.SSLContext) + mock_ctx.wrap_socket.side_effect = ssl_error + mock_raw = _make_mock_raw_sock() + with patch("socket.create_connection", return_value=mock_raw): + context.check_result = check_tls_certificate( + hostname=hostname, + warn_days=warn_days, + ssl_context=mock_ctx, + ) + elif timeout_error: + # Simulate TimeoutError during create_connection + with patch("socket.create_connection", side_effect=TimeoutError("timed out")): + context.check_result = check_tls_certificate( + hostname=hostname, + warn_days=warn_days, + ) + elif connection_error is not None: + # Simulate OSError during create_connection + with patch("socket.create_connection", side_effect=connection_error): + context.check_result = check_tls_certificate( + hostname=hostname, + warn_days=warn_days, + ) + else: + # Normal path: inject a mock certificate + cert = _make_cert( + cn=context.cert_cn, + sans=context.cert_sans, + not_after=context.cert_not_after, + ) + mock_ctx = _make_mock_ssl_context(cert) + mock_raw = _make_mock_raw_sock() + with patch("socket.create_connection", return_value=mock_raw): + context.check_result = check_tls_certificate( + hostname=hostname, + warn_days=warn_days, + ssl_context=mock_ctx, + ) + + +@when('the TLS check runs for hostname "{hostname}"') +def step_run_check(context: Context, hostname: str) -> None: + """Run the TLS check for *hostname* with default warn-days (30).""" + _run_check(context, hostname, warn_days=30) + + +@when('the TLS check runs for hostname "{hostname}" with warn-days {warn_days:d}') +def step_run_check_with_warn_days( + context: Context, hostname: str, warn_days: int +) -> None: + """Run the TLS check for *hostname* with a custom warn-days threshold.""" + _run_check(context, hostname, warn_days=warn_days) + + +@when('I check if hostname "{hostname}" matches SANs "{sans_str}"') +def step_check_san_match(context: Context, hostname: str, sans_str: str) -> None: + """Invoke _hostname_matches_san directly.""" + sans = [s.strip() for s in sans_str.split(",")] + context.san_match_result = _hostname_matches_san(hostname, sans) + + +# --------------------------------------------------------------------------- +# Then — assertions +# --------------------------------------------------------------------------- + + +@then("the check result should be a success") +def step_check_success(context: Context) -> None: + """Assert the check result is OK.""" + result: Any = context.check_result + assert result.ok, f"Expected TLS check to succeed but got errors: {result.errors!r}" + + +@then("the check result should be a failure") +def step_check_failure(context: Context) -> None: + """Assert the check result is not OK.""" + result: Any = context.check_result + assert not result.ok, ( + f"Expected TLS check to fail but it succeeded (warnings: {result.warnings!r})" + ) + + +@then('the check error should mention "{text}"') +def step_check_error_mentions(context: Context, text: str) -> None: + """Assert at least one error message contains *text*.""" + result: Any = context.check_result + combined = " ".join(result.errors) + assert text.lower() in combined.lower(), ( + f"Expected error containing {text!r} but got errors: {result.errors!r}" + ) + + +@then('the check warning should mention "{text}"') +def step_check_warning_mentions(context: Context, text: str) -> None: + """Assert at least one warning message contains *text*.""" + result: Any = context.check_result + combined = " ".join(result.warnings) + assert text.lower() in combined.lower(), ( + f"Expected warning containing {text!r} but got warnings: {result.warnings!r}" + ) + + +@then("the check has no warnings") +def step_check_no_warnings(context: Context) -> None: + """Assert the check result has no warnings.""" + result: Any = context.check_result + assert not result.warnings, f"Expected no warnings but got: {result.warnings!r}" + + +@then("the SAN match result should be True") +def step_san_match_true(context: Context) -> None: + """Assert _hostname_matches_san returned True.""" + assert context.san_match_result is True, ( + f"Expected SAN match to be True but got {context.san_match_result!r}" + ) + + +@then("the SAN match result should be False") +def step_san_match_false(context: Context) -> None: + """Assert _hostname_matches_san returned False.""" + assert context.san_match_result is False, ( + f"Expected SAN match to be False but got {context.san_match_result!r}" + ) diff --git a/features/tls_certificate_check.feature b/features/tls_certificate_check.feature new file mode 100644 index 000000000..df729e680 --- /dev/null +++ b/features/tls_certificate_check.feature @@ -0,0 +1,110 @@ +Feature: TLS certificate health-check script + The ``scripts/check-tls-cert.py`` script inspects TLS certificates for + CleverAgents infrastructure hostnames and reports errors, warnings, and + certificate metadata. These scenarios exercise the script's core logic + using injected SSL contexts so no real network connections are made. + + # ── Regression tests for issue #1543 ────────────────────────────────── + # The TLS handshake failure on git.dev.cleveragents.com was caused by the + # hostname being absent from the certificate's Subject Alternative Names + # (SANs). The scenarios below verify that the check script correctly + # detects this condition and reports it as an error. + + @tdd_issue @tdd_issue_1543 + Scenario: Script detects missing SAN for git.dev.cleveragents.com + Given a TLS certificate for "git.cleverthis.com" with SANs "git.cleverthis.com,git.cleveragents.com" + And the certificate expires in 90 days + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a failure + And the check error should mention "not found in certificate SANs" + + @tdd_issue @tdd_issue_1543 + Scenario: Script passes when hostname is present in SANs + Given a TLS certificate for "git.cleverthis.com" with SANs "git.cleverthis.com,git.dev.cleveragents.com" + And the certificate expires in 90 days + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a success + + @tdd_issue @tdd_issue_1543 + Scenario: Script detects expired certificate + Given a TLS certificate for "git.cleverthis.com" with SANs "git.cleverthis.com,git.dev.cleveragents.com" + And the certificate expired 5 days ago + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a failure + And the check error should mention "expired" + + @tdd_issue @tdd_issue_1543 + Scenario: Script warns when certificate expires within threshold + Given a TLS certificate for "git.cleverthis.com" with SANs "git.cleverthis.com,git.dev.cleveragents.com" + And the certificate expires in 15 days + When the TLS check runs for hostname "git.dev.cleveragents.com" with warn-days 30 + Then the check result should be a success + And the check warning should mention "expires in" + + @tdd_issue @tdd_issue_1543 + Scenario: Script does not warn when certificate expires beyond threshold + Given a TLS certificate for "git.cleverthis.com" with SANs "git.cleverthis.com,git.dev.cleveragents.com" + And the certificate expires in 60 days + When the TLS check runs for hostname "git.dev.cleveragents.com" with warn-days 30 + Then the check result should be a success + And the check has no warnings + + @tdd_issue @tdd_issue_1543 + Scenario: Script reports TLS handshake failure as an error + Given the TLS connection raises an SSLError "CERTIFICATE_VERIFY_FAILED" + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a failure + And the check error should mention "TLS" + + @tdd_issue @tdd_issue_1543 + Scenario: Script reports connection timeout as an error + Given the TLS connection times out + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a failure + And the check error should mention "timed out" + + @tdd_issue @tdd_issue_1543 + Scenario: Script reports connection refused as an error + Given the TLS connection is refused + When the TLS check runs for hostname "git.dev.cleveragents.com" + Then the check result should be a failure + And the check error should mention "Connection failed" + + # ── Wildcard SAN matching ────────────────────────────────────────────── + + @tdd_issue @tdd_issue_1543 + Scenario: Script accepts wildcard SAN matching the hostname + Given a TLS certificate for "git.cleverthis.com" with SANs "*.cleverthis.com" + And the certificate expires in 90 days + When the TLS check runs for hostname "git.cleverthis.com" + Then the check result should be a success + + @tdd_issue @tdd_issue_1543 + Scenario: Script rejects wildcard SAN that does not match the hostname + Given a TLS certificate for "git.cleverthis.com" with SANs "*.cleveragents.com" + And the certificate expires in 90 days + When the TLS check runs for hostname "git.cleverthis.com" + Then the check result should be a failure + And the check error should mention "not found in certificate SANs" + + # ── Helper function unit tests ───────────────────────────────────────── + + @tdd_issue @tdd_issue_1543 + Scenario: _hostname_matches_san returns True for exact match + When I check if hostname "git.dev.cleveragents.com" matches SANs "git.dev.cleveragents.com,git.cleveragents.com" + Then the SAN match result should be True + + @tdd_issue @tdd_issue_1543 + Scenario: _hostname_matches_san returns False when hostname is absent + When I check if hostname "git.dev.cleveragents.com" matches SANs "git.cleveragents.com,git.cleverthis.com" + Then the SAN match result should be False + + @tdd_issue @tdd_issue_1543 + Scenario: _hostname_matches_san handles wildcard SANs correctly + When I check if hostname "git.cleverthis.com" matches SANs "*.cleverthis.com" + Then the SAN match result should be True + + @tdd_issue @tdd_issue_1543 + Scenario: _hostname_matches_san rejects multi-level wildcard + When I check if hostname "a.b.cleverthis.com" matches SANs "*.cleverthis.com" + Then the SAN match result should be False diff --git a/mkdocs.yml b/mkdocs.yml index af970ef5b..de3458930 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Testing Guide: development/testing.md - Review Playbook: development/review_playbook.md - Scale Testing: development/scale_testing.md + - Ops Runbook: development/ops-runbook.md - Implementation Timeline: timeline.md - FAQ: faq.md - Reference: reference/ diff --git a/scripts/check-tls-cert.py b/scripts/check-tls-cert.py new file mode 100644 index 000000000..20e489876 --- /dev/null +++ b/scripts/check-tls-cert.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""TLS certificate health-check script for CleverAgents infrastructure. + +Verifies that a given hostname's TLS certificate: + - Is reachable via HTTPS + - Presents a valid certificate trusted by the system CA bundle + - Includes the target hostname as a Subject Alternative Name (SAN) + - Has not expired and will not expire within the warning threshold + +Usage: + python scripts/check-tls-cert.py git.dev.cleveragents.com + python scripts/check-tls-cert.py git.dev.cleveragents.com --port 443 + python scripts/check-tls-cert.py git.dev.cleveragents.com --warn-days 30 + python scripts/check-tls-cert.py git.dev.cleveragents.com git.cleveragents.com + +Exit codes: + 0 All checks passed + 1 One or more checks failed (certificate error, expiry, SAN mismatch) + 2 Usage error (bad arguments) + +This script is used by the ops team to diagnose TLS issues and by CI to +monitor certificate health. See docs/development/ops-runbook.md for the +full certificate renewal procedure. +""" + +from __future__ import annotations + +import argparse +import datetime +import socket +import ssl +import sys +from collections.abc import Sequence +from dataclasses import dataclass, field + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass +class CertCheckResult: + """Result of a single TLS certificate check for one hostname.""" + + hostname: str + port: int + ok: bool = True + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + subject_cn: str = "" + sans: list[str] = field(default_factory=list) + not_after: datetime.datetime | None = None + days_remaining: int | None = None + + def add_error(self, msg: str) -> None: + """Record a fatal error and mark the result as failed.""" + self.errors.append(msg) + self.ok = False + + def add_warning(self, msg: str) -> None: + """Record a non-fatal warning (does not affect ok status).""" + self.warnings.append(msg) + + +# --------------------------------------------------------------------------- +# Certificate inspection helpers +# --------------------------------------------------------------------------- + + +def _parse_not_after(cert: dict) -> datetime.datetime | None: + """Parse the ``notAfter`` field from a DER-decoded certificate dict. + + Returns a timezone-aware UTC datetime, or ``None`` if the field is + absent or cannot be parsed. + """ + not_after_str: str | None = cert.get("notAfter") + if not not_after_str: + return None + try: + # ssl module returns strings like "Apr 2 23:59:59 2026 GMT" + dt = datetime.datetime.strptime(not_after_str, "%b %d %H:%M:%S %Y %Z") + return dt.replace(tzinfo=datetime.UTC) + except ValueError: + return None + + +def _extract_sans(cert: dict) -> list[str]: + """Extract Subject Alternative Names from a DER-decoded certificate dict. + + Returns a list of SAN values (e.g. ``["git.dev.cleveragents.com", + "git.cleveragents.com"]``). DNS SANs are returned without the + ``DNS:`` prefix. + """ + sans: list[str] = [] + for san_type, san_value in cert.get("subjectAltName", ()): + if san_type == "DNS": + sans.append(san_value) + return sans + + +def _extract_cn(cert: dict) -> str: + """Extract the Common Name (CN) from the certificate subject.""" + for rdn in cert.get("subject", ()): + for attr_type, attr_value in rdn: + if attr_type == "commonName": + return attr_value + return "" + + +def _hostname_matches_san(hostname: str, sans: list[str]) -> bool: + """Return True if *hostname* matches any SAN entry. + + Supports exact matches and single-level wildcard patterns + (e.g. ``*.cleveragents.com`` matches ``git.cleveragents.com`` but + not ``a.b.cleveragents.com``). + """ + hostname_lower = hostname.lower() + for san in sans: + san_lower = san.lower() + if san_lower == hostname_lower: + return True + if san_lower.startswith("*."): + # Wildcard: *.example.com matches foo.example.com only + suffix = san_lower[1:] # ".example.com" + if ( + hostname_lower.endswith(suffix) + and "." not in hostname_lower[: -len(suffix)] + ): + return True + return False + + +# --------------------------------------------------------------------------- +# Core check function +# --------------------------------------------------------------------------- + + +def check_tls_certificate( + hostname: str, + port: int = 443, + warn_days: int = 30, + timeout: float = 10.0, + ssl_context: ssl.SSLContext | None = None, +) -> CertCheckResult: + """Perform a full TLS certificate check for *hostname*:*port*. + + Args: + hostname: The hostname to connect to and verify. + port: TCP port (default 443). + warn_days: Warn if the certificate expires within this many days. + timeout: Socket connection timeout in seconds. + ssl_context: Optional custom SSLContext (used in tests to inject + mock behaviour). When ``None``, a default context + with system CA verification is used. + + Returns: + A :class:`CertCheckResult` describing the outcome. + """ + result = CertCheckResult(hostname=hostname, port=port) + + # Build default SSL context if none provided + ctx = ssl.create_default_context() if ssl_context is None else ssl_context + + # --- Attempt TLS handshake --- + try: + with ( + socket.create_connection((hostname, port), timeout=timeout) as raw_sock, + ctx.wrap_socket(raw_sock, server_hostname=hostname) as tls_sock, + ): + cert = tls_sock.getpeercert() + except ssl.SSLCertVerificationError as exc: + result.add_error(f"TLS verification failed: {exc.reason}") + return result + except ssl.SSLError as exc: + result.add_error(f"TLS handshake error: {exc}") + return result + except TimeoutError: + result.add_error(f"Connection timed out after {timeout}s") + return result + except OSError as exc: + result.add_error(f"Connection failed: {exc}") + return result + + if cert is None: + result.add_error("No certificate returned by server") + return result + + # --- Parse certificate fields --- + result.subject_cn = _extract_cn(cert) + result.sans = _extract_sans(cert) + result.not_after = _parse_not_after(cert) + + # --- SAN check --- + if not _hostname_matches_san(hostname, result.sans): + result.add_error( + f"Hostname '{hostname}' not found in certificate SANs: " + f"{result.sans!r}. The certificate may be missing this hostname " + "or SNI routing is misconfigured." + ) + + # --- Expiry check --- + if result.not_after is not None: + now = datetime.datetime.now(tz=datetime.UTC) + delta = result.not_after - now + result.days_remaining = delta.days + if delta.total_seconds() <= 0: + expiry = result.not_after.strftime("%Y-%m-%d %H:%M:%S UTC") + result.add_error(f"Certificate expired on {expiry}") + elif delta.days <= warn_days: + expiry_date = result.not_after.strftime("%Y-%m-%d") + result.add_warning( + f"Certificate expires in {delta.days} day(s) " + f"({expiry_date}). " + f"Renewal recommended (threshold: {warn_days} days)." + ) + else: + result.add_warning("Could not determine certificate expiry date") + + return result + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + + +def _print_result(result: CertCheckResult) -> None: + """Print a human-readable summary of a :class:`CertCheckResult`.""" + status = "OK" if result.ok else "FAIL" + print(f"\n[{status}] {result.hostname}:{result.port}") + if result.subject_cn: + print(f" Subject CN : {result.subject_cn}") + if result.sans: + print(f" SANs : {', '.join(result.sans)}") + if result.not_after: + days = result.days_remaining + expiry = result.not_after.strftime("%Y-%m-%d %H:%M:%S UTC") + print(f" Expires : {expiry} ({days} day(s) remaining)") + for err in result.errors: + print(f" ERROR : {err}") + for warn in result.warnings: + print(f" WARNING : {warn}") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse arguments and run TLS checks. Returns the process exit code.""" + parser = argparse.ArgumentParser( + description="Check TLS certificate validity for one or more hostnames.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "hostnames", + nargs="+", + metavar="HOSTNAME", + help="One or more hostnames to check (e.g. git.dev.cleveragents.com)", + ) + parser.add_argument( + "--port", + type=int, + default=443, + help="HTTPS port to connect to (default: 443)", + ) + parser.add_argument( + "--warn-days", + type=int, + default=30, + metavar="DAYS", + help="Warn if certificate expires within DAYS days (default: 30)", + ) + parser.add_argument( + "--timeout", + type=float, + default=10.0, + metavar="SECONDS", + help="Connection timeout in seconds (default: 10)", + ) + + args = parser.parse_args(argv) + + all_ok = True + for hostname in args.hostnames: + result = check_tls_certificate( + hostname=hostname, + port=args.port, + warn_days=args.warn_days, + timeout=args.timeout, + ) + _print_result(result) + if not result.ok: + all_ok = False + + print() + if all_ok: + print("All TLS checks PASSED") + return 0 + else: + print("One or more TLS checks FAILED") + return 1 + + +if __name__ == "__main__": + sys.exit(main())