From 38c60265cc1a769d2ec7ee06ab91641ba9c6df7e Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Fri, 15 May 2026 13:55:56 -0700 Subject: [PATCH 1/6] saving user-lookup client --- app/s3df/clients/__init__.py | 3 ++ app/s3df/clients/user_lookup.py | 73 +++++++++++++++++++++++++ app/s3df/config.py | 3 ++ app/s3df/filesystem_adapter.py | 95 +++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 app/s3df/clients/user_lookup.py create mode 100644 app/s3df/filesystem_adapter.py diff --git a/app/s3df/clients/__init__.py b/app/s3df/clients/__init__.py index ba2766de..ace42855 100644 --- a/app/s3df/clients/__init__.py +++ b/app/s3df/clients/__init__.py @@ -5,8 +5,11 @@ """ from app.s3df.clients.coact import CoactClient, get_coact_client +from app.s3df.clients.user_lookup import UserLookupClient, get_user_lookup_client __all__ = [ "CoactClient", "get_coact_client", + "UserLookupClient", + "get_user_lookup_client", ] diff --git a/app/s3df/clients/user_lookup.py b/app/s3df/clients/user_lookup.py new file mode 100644 index 00000000..fee58d51 --- /dev/null +++ b/app/s3df/clients/user_lookup.py @@ -0,0 +1,73 @@ +""" +user-lookup GraphQL Client + +Async client for querying the user-lookup service for POSIX identity data. +No authentication required (user-lookup has open CORS). +""" + +import logging +from typing import Optional + +from gql import gql, Client +from gql.transport.httpx import HTTPXAsyncTransport + +from app.s3df.config import settings + +LOG = logging.getLogger(__name__) + +_GET_USER_QUERY = gql(""" + query GetUser($filter: UserInput!) { + users(filter: $filter) { + username + uidnumber + gidNumber + secondaryGidNumbers + } + } +""") + + +class UserLookupClient: + """GraphQL client for the user-lookup service.""" + + def __init__(self, url: str | None = None): + self.url = (url or settings.user_lookup_url).rstrip("/") + "/graphql" + LOG.info(f"Initialized UserLookupClient for endpoint: {self.url}") + + def _get_client(self) -> Client: + transport = HTTPXAsyncTransport( + url=self.url, + headers={"Content-Type": "application/json"}, + timeout=10.0, + ) + return Client(transport=transport, fetch_schema_from_transport=False) + + async def get_user(self, username: str) -> dict: + """ + Fetch user POSIX identity from user-lookup. + + Returns dict with: username, uidnumber, gidNumber, secondaryGidNumbers. + Raises ValueError if user not found, or propagates transport errors + if the service is unreachable. + """ + client = self._get_client() + async with client as session: + result = await session.execute( + _GET_USER_QUERY, + variable_values={"filter": {"username": username}}, + ) + users = result.get("users", []) + if not users: + raise ValueError(f"User '{username}' not found in user-lookup") + return users[0] + + +_default_client: Optional[UserLookupClient] = None + + +def get_user_lookup_client() -> UserLookupClient: + """Get or create the singleton UserLookupClient instance.""" + global _default_client + if _default_client is None: + _default_client = UserLookupClient() + return _default_client diff --git a/app/s3df/config.py b/app/s3df/config.py index c0236886..50796253 100644 --- a/app/s3df/config.py +++ b/app/s3df/config.py @@ -33,5 +33,8 @@ def __init__(self): self.dex_audience = os.getenv("DEX_AUDIENCE") self.dex_username_claim = os.getenv("DEX_USERNAME_CLAIM", "name") + # user-lookup service (direct LDAP/POSIX identity queries) + self.user_lookup_url = os.getenv("USER_LOOKUP_URL", "") + settings = S3DFSettings() diff --git a/app/s3df/filesystem_adapter.py b/app/s3df/filesystem_adapter.py new file mode 100644 index 00000000..d3db4c08 --- /dev/null +++ b/app/s3df/filesystem_adapter.py @@ -0,0 +1,95 @@ +""" +S3DF Filesystem Adapter + +Thin proxy adapter for filesystem operations. Authenticates via Dex JWT, +enriches with POSIX identity from user-lookup, and will forward requests +to the filesystem microservice (same endpoints, same data model). +""" + +import logging + +from fastapi import HTTPException + +from app.types.user import User +from app.s3df.auth.authenticated_adapter import S3DFAuthenticatedAdapter +from app.s3df.clients import get_user_lookup_client +from app.routers.filesystem import facility_adapter, models +from app.routers.status import models as status_models + +LOG = logging.getLogger(__name__) + + +class S3DFFilesystemAdapter(S3DFAuthenticatedAdapter, facility_adapter.FacilityAdapter): + """Filesystem adapter that enforces POSIX identity via user-lookup.""" + + async def get_user(self, user_id: str, api_key: str, client_ip: str | None, globus_introspect: dict | None = None) -> User: + try: + lookup_data = await get_user_lookup_client().get_user(user_id) + except ValueError: + raise HTTPException(status_code=403, detail=f"User '{user_id}' not found in directory") + except Exception as e: + LOG.error(f"user-lookup service error: {e}") + raise HTTPException(status_code=502, detail="User lookup service unavailable") + + return User( + id=user_id, + name=lookup_data.get("username", user_id), + api_key=api_key, + client_ip=client_ip, + ) + + # --- Filesystem operations (501 until microservice is connected) --- + + async def chmod(self, resource: status_models.Resource, user: User, request_model: models.PutFileChmodRequest) -> models.PutFileChmodResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def chown(self, resource: status_models.Resource, user: User, request_model: models.PutFileChownRequest) -> models.PutFileChownResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def ls(self, resource: status_models.Resource, user: User, path: str, show_hidden: bool, numeric_uid: bool, recursive: bool, dereference: bool) -> models.GetDirectoryLsResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def head(self, resource: status_models.Resource, user: User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> models.GetFileHeadResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def tail(self, resource: status_models.Resource, user: User, path: str, file_bytes: int | None, lines: int | None, skip_heading: bool) -> models.GetFileTailResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def view(self, resource: status_models.Resource, user: User, path: str, size: int, offset: int) -> models.GetViewFileResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def checksum(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileChecksumResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def file(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileTypeResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def stat(self, resource: status_models.Resource, user: User, path: str, dereference: bool) -> models.GetFileStatResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def rm(self, resource: status_models.Resource, user: User, path: str) -> models.RemoveResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def mkdir(self, resource: status_models.Resource, user: User, request_model: models.PostMakeDirRequest) -> models.PostMkdirResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def symlink(self, resource: status_models.Resource, user: User, request_model: models.PostFileSymlinkRequest) -> models.PostFileSymlinkResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def download(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileDownloadResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def upload(self, resource: status_models.Resource, user: User, path: str, content: str) -> models.PutFileUploadResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def compress(self, resource: status_models.Resource, user: User, request_model: models.PostCompressRequest) -> models.PostCompressResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def extract(self, resource: status_models.Resource, user: User, request_model: models.PostExtractRequest) -> models.PostExtractResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def mv(self, resource: status_models.Resource, user: User, request_model: models.PostMoveRequest) -> models.PostMoveResponse: + raise HTTPException(status_code=501, detail="Not implemented") + + async def cp(self, resource: status_models.Resource, user: User, request_model: models.PostCopyRequest) -> models.PostCopyResponse: + raise HTTPException(status_code=501, detail="Not implemented") From f6a5e03bbd7ec5b8bcfdbdfbcac5e4871c93c5d4 Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Fri, 29 May 2026 21:06:26 -0500 Subject: [PATCH 2/6] status adapter and jwt_verifier accepts lists of audience --- Dockerfile | 1 + Makefile | 1 + app/s3df/__init__.py | 3 +- app/s3df/auth/jwt_verifier.py | 4 +- app/s3df/config.py | 6 +- app/s3df/status_adapter.py | 637 ++++++++++++++++++++++++ design-docs/coact-user-impersonation.md | 119 +++++ design-docs/debug-facility-nersc.md | 149 ++++++ design-docs/s3df-status-adapter.md | 329 ++++++++++++ 9 files changed, 1245 insertions(+), 4 deletions(-) create mode 100644 app/s3df/status_adapter.py create mode 100644 design-docs/coact-user-impersonation.md create mode 100644 design-docs/debug-facility-nersc.md create mode 100644 design-docs/s3df-status-adapter.md diff --git a/Dockerfile b/Dockerfile index 15fa10d0..266fef42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN pip install -U pip wheel setuptools uv && \ uv pip install --system . ENV IRI_API_ADAPTER_account="app.s3df.account_adapter.S3DFAccountAdapter" +ENV IRI_API_ADAPTER_status="app.s3df.status_adapter.S3DFStatusAdapter" ENV IRI_SHOW_MISSING_ROUTES="true" ENV DEX_JWKS_URL="https://dex.slac.stanford.edu/keys" ENV DEX_ISSUER="https://dex.slac.stanford.edu" diff --git a/Makefile b/Makefile index 98caa55e..dae14644 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ dev: deps dev-s3df: deps @source $(BIN)/activate && \ IRI_API_ADAPTER_account=app.s3df.account_adapter.S3DFAccountAdapter \ + IRI_API_ADAPTER_status=app.s3df.status_adapter.S3DFStatusAdapter \ COACT_API_URL='https://coact-dev.slac.stanford.edu/graphql-service-dev' \ IRI_SHOW_MISSING_ROUTES='true' \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/app/s3df/__init__.py b/app/s3df/__init__.py index af035aea..e8babb8f 100644 --- a/app/s3df/__init__.py +++ b/app/s3df/__init__.py @@ -8,6 +8,7 @@ from .account_adapter import S3DFAccountAdapter from .facility_adapter import S3DFFacilityAdapter from .compute_adapter import SLACComputeAdapter +from .status_adapter import S3DFStatusAdapter from .config import settings -__all__ = ["S3DFAccountAdapter", "S3DFFacilityAdapter", "SLACComputeAdapter", "settings"] +__all__ = ["S3DFAccountAdapter", "S3DFFacilityAdapter", "SLACComputeAdapter", "S3DFStatusAdapter", "settings"] diff --git a/app/s3df/auth/jwt_verifier.py b/app/s3df/auth/jwt_verifier.py index a597a896..68780c2d 100644 --- a/app/s3df/auth/jwt_verifier.py +++ b/app/s3df/auth/jwt_verifier.py @@ -11,7 +11,7 @@ import asyncio import logging import ssl -from typing import Optional +from typing import List, Optional, Union import jwt from fastapi import HTTPException @@ -29,7 +29,7 @@ def __init__( self, jwks_url: str, issuer: str, - audience: str, + audience: Union[str, List[str]], username_claim: str = "name", ): self.jwks_url = jwks_url diff --git a/app/s3df/config.py b/app/s3df/config.py index 50796253..207052e6 100644 --- a/app/s3df/config.py +++ b/app/s3df/config.py @@ -30,7 +30,11 @@ def __init__(self): # Dex IdP — JWKS-based JWT verification for incoming Bearer tokens. self.dex_jwks_url = os.getenv("DEX_JWKS_URL") self.dex_issuer = os.getenv("DEX_ISSUER") - self.dex_audience = os.getenv("DEX_AUDIENCE") + # Comma-separated list of accepted audiences (e.g. "aud1,aud2"). + _raw_audience = os.getenv("DEX_AUDIENCE", "") + self.dex_audience: list[str] = [ + a.strip() for a in _raw_audience.split(",") if a.strip() + ] self.dex_username_claim = os.getenv("DEX_USERNAME_CLAIM", "name") # user-lookup service (direct LDAP/POSIX identity queries) diff --git a/app/s3df/status_adapter.py b/app/s3df/status_adapter.py new file mode 100644 index 00000000..92d222d2 --- /dev/null +++ b/app/s3df/status_adapter.py @@ -0,0 +1,637 @@ +""" +SLAC S3DF Status Adapter for the IRI Facility API. + +Serves the IRI ``/status`` router (resources, events, incidents) from live +health checks against S3DF monitoring infrastructure — Prometheus and InfluxDB +(telegraf) — the same backends the standalone ``status-pusher`` tool queries. + +Design (see design-docs/s3df-status-adapter.md): + + * Resources are STATIC config (``RESOURCE_REGISTRY``). Each carries a health + check: a backend + query + condition (comparator, threshold) — mirroring the + success-criterion model used by ``status-pusher``. + * A background poller runs every ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries + each resource's backend, maps the result to a Status (up/down/degraded/ + unknown), and records it in an in-memory store. + * The store detects status TRANSITIONS to emit Events and to open/close + unplanned Incidents. + +Lifecycle note: IRI constructs adapters synchronously at import/router-build time +— before the asyncio event loop exists — so the poller and its ``httpx`` client +are created LAZILY on the first request (``_ensure_started``), where the loop is +running. State is in-memory and per-process: it is lost on restart and diverges +across multiple uvicorn workers. Persistence and FastAPI-lifespan shutdown wiring +are tracked as follow-ups in the design doc. + +Required/optional env vars: + S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) + S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) + S3DF_INFLUXDB_DB InfluxDB database (default telegraf) + S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) + S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) + S3DF_SITE_ID Site id for resources (default s3df) + S3DF_STATUS_TLS_VERIFY true | false | (default false) +""" + +import asyncio +import datetime +import logging +import operator +import os +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import Callable + +import httpx + +from app.routers.status import facility_adapter as status_adapter +from app.routers.status import models as status_models +from app.routers.status.models import ( + Event, + Incident, + IncidentType, + Resolution, + Resource, + ResourceType, + Status, +) + +logger = logging.getLogger(__name__) + + +def _utc_now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +# --------------------------------------------------------------------------- +# Configuration / resource registry +# --------------------------------------------------------------------------- + + +class Backend(str, Enum): + """Metrics backend a health check queries.""" + + prometheus = "prometheus" + influxdb = "influxdb" + + +_COMPARATORS: dict[str, Callable[[float, float], bool]] = { + "eq": operator.eq, + "ne": operator.ne, + "lt": operator.lt, + "lte": operator.le, + "gt": operator.gt, + "gte": operator.ge, +} + + +@dataclass(frozen=True) +class Condition: + """A comparison of an observed metric value against a threshold.""" + + comparator: str + value: float + + def met(self, observed: float) -> bool: + op = _COMPARATORS.get(self.comparator) + if op is None: + raise ValueError(f"Unknown comparator: {self.comparator}") + return bool(op(observed, self.value)) + + +@dataclass(frozen=True) +class HealthCheck: + """How to determine a resource's status from a metrics backend.""" + + backend: Backend + query: str + up_when: Condition + db_name: str | None = None # InfluxDB only + degraded_when: Condition | None = None + + +@dataclass(frozen=True) +class ResourceDef: + """Static definition of a monitored resource + its health check.""" + + id: str + name: str + description: str + group: str + resource_type: ResourceType + health_check: HealthCheck + capability_ids: tuple[str, ...] = () + + +class StatusSettings: + """Environment-driven settings for the S3DF status adapter.""" + + def __init__(self) -> None: + self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") + self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") + self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") + self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) + self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) + # NOTE: the /facility adapter currently mints a random site uuid per + # process, so this id is not a guaranteed cross-reference yet. Set + # S3DF_SITE_ID once a stable site identifier is established. + self.site_id = os.getenv("S3DF_SITE_ID", "s3df") + self.tls_verify = self._parse_verify(os.getenv("S3DF_STATUS_TLS_VERIFY", "false")) + + @staticmethod + def _parse_verify(raw: str) -> bool | str: + low = raw.strip().lower() + if low in ("true", "1", "yes", "on"): + return True + if low in ("false", "0", "no", "off", ""): + return False + # Anything else is treated as a path to a CA bundle. + return raw + + +# The set of S3DF resources surfaced by /status. Queries mirror those exercised +# by status-pusher's live tests (see status-pusher/Makefile). Extend as needed. +RESOURCE_REGISTRY: list[ResourceDef] = [ + ResourceDef( + id="s3df-ssh-gateway", + name="SSH Login Gateway", + description="S3DF interactive SSH login gateway reachability.", + group="access", + resource_type=ResourceType.service, + health_check=HealthCheck( + backend=Backend.prometheus, + query="avg( avg_over_time(nmap_port_state{service=`ssh`,group=`s3df`}[5m]) )", + up_when=Condition("eq", 1.0), + ), + ), + ResourceDef( + id="s3df-slurmctld", + name="Slurm Controller (slurmctld)", + description="Slurm workload manager controller daemon health.", + group="compute", + resource_type=ResourceType.compute, + health_check=HealthCheck( + backend=Backend.influxdb, + db_name="telegraf", + query=( + 'SELECT mean("status_code") FROM "monit_process" ' + "WHERE \"service\" = 'slurmctld' AND time > now()-5m" + ), + up_when=Condition("eq", 1.0), + ), + ), + ResourceDef( + id="s3df-slurmdbd", + name="Slurm DB Daemon (slurmdbd)", + description="Slurm accounting database daemon health.", + group="compute", + resource_type=ResourceType.compute, + health_check=HealthCheck( + backend=Backend.influxdb, + db_name="telegraf", + query=( + 'SELECT mean("status_code") FROM "monit_process" ' + "WHERE \"service\" = 'slurmdbd' AND time > now()-5m" + ), + up_when=Condition("eq", 1.0), + ), + ), +] + + +# --------------------------------------------------------------------------- +# Health checker — async Prometheus / InfluxDB queries + evaluation +# --------------------------------------------------------------------------- + + +@dataclass +class HealthResult: + """Outcome of a single health check.""" + + status: Status + value: float | None + observed_at: datetime.datetime + error: str | None = None + + +def evaluate(check: HealthCheck, value: float | None) -> Status: + """Map an observed metric value to a Status using the check's conditions.""" + if value is None: + return Status.unknown + if check.up_when.met(value): + return Status.up + if check.degraded_when is not None and check.degraded_when.met(value): + return Status.degraded + return Status.down + + +class HealthChecker: + """Runs a HealthCheck against its backend and evaluates the result. + + The caller owns the ``httpx.AsyncClient`` lifecycle (created from within the + running event loop) and passes it in. + """ + + def __init__(self, settings: StatusSettings, client: httpx.AsyncClient): + self.settings = settings + self.client = client + + async def check(self, check: HealthCheck) -> HealthResult: + now = _utc_now() + try: + if check.backend == Backend.prometheus: + value = await self._prometheus_query(check.query) + elif check.backend == Backend.influxdb: + db = check.db_name or self.settings.influxdb_db + value = await self._influx_query(db, check.query) + else: # pragma: no cover - guarded by enum + return HealthResult(Status.unknown, None, now, f"unknown backend {check.backend}") + except Exception as exc: # noqa: BLE001 - any failure -> unknown + logger.warning("Health check failed (%s): %s", check.backend.value, exc) + return HealthResult(Status.unknown, None, now, str(exc)) + return HealthResult(evaluate(check, value), value, now) + + async def _prometheus_query(self, query: str) -> float | None: + """Instant query via the Prometheus HTTP API; returns the scalar value.""" + url = self.settings.prometheus_url.rstrip("/") + "/api/v1/query" + resp = await self.client.get(url, params={"query": query}, timeout=self.settings.http_timeout) + resp.raise_for_status() + data = resp.json() + if data.get("status") != "success": + raise RuntimeError(f"prometheus error: {data.get('error', 'unknown')}") + result = data.get("data", {}).get("result", []) + if not result: + return None + # Instant vector sample: value == [epoch_ts, "stringified_value"] + return float(result[0]["value"][1]) + + async def _influx_query(self, db_name: str, query: str) -> float | None: + """Query the InfluxDB HTTP API; returns the most recent scalar value.""" + url = self.settings.influxdb_url.rstrip("/") + "/query" + resp = await self.client.get( + url, params={"q": query, "db": db_name}, timeout=self.settings.http_timeout + ) + resp.raise_for_status() + data = resp.json() + results = data.get("results", []) + if not results: + return None + series = results[0].get("series") + if not series: + return None + values = series[0].get("values") + if not values: + return None + # Each row is [time, value]; take the latest row's value column. + row = values[-1] + return float(row[1]) if row[1] is not None else None + + +# --------------------------------------------------------------------------- +# In-memory store — current status, events, incidents +# --------------------------------------------------------------------------- + + +class StatusStore: + """Holds current status per resource plus an event log and incident set. + + Single-writer model: only the poller calls ``record()``, and ``record()`` has + no internal ``await`` points, so each mutation is atomic with respect to the + asyncio event loop. Reader methods return freshly built/copied lists, so a + request handler never observes a half-applied update. State is per-process and + volatile. + """ + + def __init__(self, site_id: str, resource_defs: list[ResourceDef]): + self.site_id = site_id + self._defs: dict[str, ResourceDef] = {d.id: d for d in resource_defs} + self._created_at = _utc_now() + self._status: dict[str, Status] = {} + self._last_modified: dict[str, datetime.datetime] = {} + self._events: list[Event] = [] + self._incidents: dict[str, Incident] = {} + self._open_incident: dict[str, str] = {} # resource_id -> incident_id + + # -- writer ------------------------------------------------------------ + + def record(self, resource_id: str, result: HealthResult) -> None: + """Record a poll result, emitting an Event + Incident change on transition.""" + if resource_id not in self._defs: + return + prev = self._status.get(resource_id) + new = result.status + ts = result.observed_at + + if prev == new: + return # steady state — nothing to record + + baseline = prev is None + self._status[resource_id] = new + self._last_modified[resource_id] = ts + + event = self._make_event(resource_id, new, ts, baseline) + + if new in (Status.down, Status.degraded): + inc_id = self._open_incident.get(resource_id) + if inc_id is None: + incident = self._make_incident(resource_id, new, ts) + self._incidents[incident.id] = incident + self._open_incident[resource_id] = incident.id + inc_id = incident.id + else: + incident = self._incidents[inc_id] + incident.status = new + incident.last_modified = ts + event.incident_id = inc_id + self._incidents[inc_id].event_ids.append(event.id) + elif new == Status.up: + inc_id = self._open_incident.pop(resource_id, None) + if inc_id is not None: + incident = self._incidents[inc_id] + incident.status = Status.up + incident.end = ts + incident.resolution = Resolution.completed + incident.last_modified = ts + event.incident_id = inc_id + incident.event_ids.append(event.id) + # Status.unknown: record the event but never open/close incidents — + # a monitoring-backend failure is not a confirmed resource outage. + + self._events.append(event) + + def _make_event(self, resource_id: str, status: Status, ts: datetime.datetime, baseline: bool) -> Event: + rdef = self._defs[resource_id] + verb = "initial status" if baseline else "status changed to" + return Event( + id=str(uuid.uuid4()), + name=f"{rdef.name}: {status.value}", + description=f"{rdef.name} {verb} {status.value}.", + last_modified=ts, + occurred_at=ts, + status=status, + resource_id=resource_id, + ) + + def _make_incident(self, resource_id: str, status: Status, ts: datetime.datetime) -> Incident: + rdef = self._defs[resource_id] + return Incident( + id=str(uuid.uuid4()), + name=f"{rdef.name} {status.value}", + description=f"Automatically opened: {rdef.name} observed {status.value}.", + last_modified=ts, + status=status, + start=ts, + type=IncidentType.unplanned, + resolution=Resolution.unresolved, + resource_ids=[resource_id], + event_ids=[], + ) + + # -- readers ----------------------------------------------------------- + + def resources(self) -> list[Resource]: + out: list[Resource] = [] + for d in self._defs.values(): + out.append( + Resource( + id=d.id, + name=d.name, + description=d.description, + last_modified=self._last_modified.get(d.id, self._created_at), + site_id=self.site_id, + group=d.group, + resource_type=d.resource_type, + current_status=self._status.get(d.id, Status.unknown), + capability_ids=list(d.capability_ids), + ) + ) + return out + + def events(self) -> list[Event]: + return sorted(self._events, key=lambda e: (e.occurred_at, e.id)) + + def incidents(self) -> list[Incident]: + return sorted(self._incidents.values(), key=lambda i: (i.start, i.id)) + + +# --------------------------------------------------------------------------- +# Background poller +# --------------------------------------------------------------------------- + + +class StatusPoller: + """Periodically runs health checks and feeds results into the store.""" + + def __init__(self, store: StatusStore, settings: StatusSettings, resource_defs: list[ResourceDef]): + self.store = store + self.settings = settings + self.resource_defs = resource_defs + self._client: httpx.AsyncClient | None = None + self._checker: HealthChecker | None = None + self._task: asyncio.Task | None = None + + async def start(self) -> None: + """Create the HTTP client (inside the running loop), run one initial poll, + then launch the periodic background loop.""" + self._client = httpx.AsyncClient(verify=self.settings.tls_verify) + self._checker = HealthChecker(self.settings, self._client) + await self._poll_once() # bounded by per-query timeouts; never raises + self._task = asyncio.create_task(self._run(), name="s3df-status-poller") + logger.info( + "S3DF status poller started (%d resources, interval=%ss)", + len(self.resource_defs), + self.settings.poll_interval, + ) + + async def _run(self) -> None: + try: + while True: + await asyncio.sleep(self.settings.poll_interval) + await self._poll_once() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - keep the server alive if the loop dies + logger.exception("S3DF status poller loop crashed") + + async def _poll_once(self) -> None: + assert self._checker is not None + defs = self.resource_defs + results = await asyncio.gather( + *(self._checker.check(d.health_check) for d in defs), + return_exceptions=True, + ) + for d, res in zip(defs, results): + if isinstance(res, Exception): + logger.warning("Health check raised for %s: %s", d.id, res) + continue + self.store.record(d.id, res) + + async def aclose(self) -> None: + """Cancel the loop and close the HTTP client. For tests/lifespan wiring.""" + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + if self._client is not None: + await self._client.aclose() + self._client = None + + +# --------------------------------------------------------------------------- +# The adapter +# --------------------------------------------------------------------------- + + +def _paginate(items: list, offset: int | None, limit: int | None) -> list: + if offset and offset > 0: + items = items[offset:] + if limit is not None and limit >= 0: + items = items[:limit] + return items + + +class S3DFStatusAdapter(status_adapter.FacilityAdapter): + """IRI status FacilityAdapter backed by live S3DF health checks. + + Implements every method called by the IRI status router: + get_resources, get_resource, get_events, get_event, + get_incidents, get_incident. + + The status router is unauthenticated, so these methods take no user. + """ + + def __init__(self, settings: StatusSettings | None = None, resource_defs: list[ResourceDef] | None = None): + self._settings = settings or StatusSettings() + self._resource_defs = resource_defs if resource_defs is not None else RESOURCE_REGISTRY + self._store = StatusStore(site_id=self._settings.site_id, resource_defs=self._resource_defs) + self._poller = StatusPoller(self._store, self._settings, self._resource_defs) + self._started = False + self._start_lock = asyncio.Lock() + + async def _ensure_started(self) -> None: + """Lazily start the poller on first use (double-checked under a lock).""" + if self._started: + return + async with self._start_lock: + if self._started: + return + try: + await self._poller.start() + self._started = True + except Exception: # noqa: BLE001 - retry on the next request + logger.exception("Failed to start S3DF status poller; will retry") + raise + + async def aclose(self) -> None: + """Release background resources (HTTP client + poller task).""" + await self._poller.aclose() + self._started = False + + # -- resources --------------------------------------------------------- + + async def get_resources( + self, + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + group: str | None = None, + modified_since: datetime.datetime | None = None, + resource_type: status_models.ResourceType | None = None, + current_status: status_models.Status | None = None, + capability=None, + site_id: str | None = None, + ) -> list[status_models.Resource]: + await self._ensure_started() + items = status_models.Resource.find( + self._store.resources(), + name=name, + description=description, + modified_since=modified_since, + group=group, + resource_type=resource_type, + current_status=current_status, + capability=capability, + site_id=site_id, + ) + return _paginate(items, offset, limit) + + async def get_resource(self, id_: str) -> status_models.Resource: + await self._ensure_started() + return status_models.Resource.find_by_id(self._store.resources(), id_, allow_name=True) + + # -- events ------------------------------------------------------------ + + async def get_events( + self, + offset: int, + limit: int, + incident_id: str | None = None, + resource_id: str | None = None, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + ) -> list[status_models.Event]: + await self._ensure_started() + items = status_models.Event.find( + self._store.events(), + incident_id=incident_id, + resource_id=resource_id, + name=name, + description=description, + status=status, + from_=from_, + to=to, + time_=time_, + modified_since=modified_since, + ) + return _paginate(items, offset, limit) + + async def get_event(self, id_: str) -> status_models.Event: + await self._ensure_started() + return status_models.Event.find_by_id(self._store.events(), id_) + + # -- incidents --------------------------------------------------------- + + async def get_incidents( + self, + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + type_: status_models.IncidentType | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + resource_id: str | None = None, + resolution: status_models.Resolution | None = None, + ) -> list[status_models.Incident]: + await self._ensure_started() + items = status_models.Incident.find( + self._store.incidents(), + name=name, + description=description, + status=status, + type_=type_, + from_=from_, + to=to, + time_=time_, + modified_since=modified_since, + resource_id=resource_id, + resolution=resolution, + ) + return _paginate(items, offset, limit) + + async def get_incident(self, id_: str) -> status_models.Incident: + await self._ensure_started() + return status_models.Incident.find_by_id(self._store.incidents(), id_) diff --git a/design-docs/coact-user-impersonation.md b/design-docs/coact-user-impersonation.md new file mode 100644 index 00000000..74ca4ddd --- /dev/null +++ b/design-docs/coact-user-impersonation.md @@ -0,0 +1,119 @@ +# Change Doc: Use coact-api User Impersonation for Authorization + +## Problem + +`CoactClient.get_user_repos(username)` currently fetches **every** repo from coact-api using the service account and then filters client-side inside iri-api: + +```python +# clients/coact.py — current (wrong) approach +async def get_user_repos(self, username: str) -> List[Dict[str, Any]]: + repos = await self.get_all_repos() # fetches ALL repos + user_repos = [repo for repo in repos + if username in repo.get("users", []) + or username in repo.get("leaders", []) + or username == repo.get("principal")] + return user_repos +``` + +This means: +- **Authorization is enforced in the wrong layer** — iri-api is doing what coact-api should be responsible for. +- Every call pulls the entire repo collection regardless of how many repos a user actually belongs to. +- The existing user-impersonation feature built into coact-api is unused. + +## Root Cause + +`CoactClient._get_client()` silently ignores the `username` parameter when `use_basic_auth=True`: + +```python +def _get_client(self, username: Optional[str] = None) -> Client: + if self.use_basic_auth: + # Only sets Authorization: Basic — username is never used + headers["Authorization"] = f"Basic {credentials}" + else: + # coactimp is only set in the non-basic-auth branch + headers["coactimp"] = username or "null" +``` + +The default `CoactClient` (returned by `get_coact_client()`) is created with `use_basic_auth=True`, so the `coactimp` impersonation header is never sent in practice, even though methods like `get_my_repos(username)` pass a username through to `execute_query` → `_get_client`. + +## How coact-api Impersonation Works + +In `coact-api/main.py` `CustomContext.authn()`: + +1. The service account authenticates as a bot/admin user. +2. If the request also includes `coactimp: `, coact-api switches its execution context to that user. +3. `is_impersonating = True` is set, which forces `isadmin = False` on all queries. +4. The `myRepos` query (and the `repos` query) then only return repos where the impersonated user appears in `users`, `leaders`, or `principal` — **server-side filtering**. + +The `isImpersonating` field is exposed via `whoami` and `User` GraphQL types confirming the impersonation is active. + +## Required Changes + +### Change 1 — `app/s3df/clients/coact.py`: `_get_client()` + +When `use_basic_auth=True` **and** a `username` is provided, also set the `coactimp` header so coact-api impersonates that user while authenticating the service account via Basic auth. + +```python +# Before +if self.use_basic_auth: + credentials = base64.b64encode(f"{self.service_user}:{self.service_password}".encode()).decode("ascii") + headers["Authorization"] = f"Basic {credentials}" + +# After +if self.use_basic_auth: + credentials = base64.b64encode(f"{self.service_user}:{self.service_password}".encode()).decode("ascii") + headers["Authorization"] = f"Basic {credentials}" + if username: + headers["coactimp"] = username # ← impersonate the target user +``` + +### Change 2 — `app/s3df/clients/coact.py`: `get_user_repos()` + +Replace the fetch-all + client-side filter with a call to the already-existing `get_my_repos(username)` method. With Change 1 in place, `get_my_repos` will now correctly set `coactimp` and coact-api will return only the user's repos. + +```python +# Before +async def get_user_repos(self, username: str) -> List[Dict[str, Any]]: + repos = await self.get_all_repos() + user_repos = [repo for repo in repos + if username in repo.get("users", []) + or username in repo.get("leaders", []) + or username == repo.get("principal")] + return user_repos + +# After +async def get_user_repos(self, username: str) -> List[Dict[str, Any]]: + return await self.get_my_repos(username) +``` + +### Change 3 — `app/s3df/account_adapter.py`: `get_projects()` + +Remove the stale commented-out membership check that was left from a previous iteration. It is now fully superseded by server-side impersonation. Update the docstring to reflect the new delegation model. + +```python +# Before (in get_projects loop) +for repo in repos: + all_users = set(repo.get("users", []) + repo.get("leaders", []) + [repo.get("principal", "")]) + # if user.id in all_users: ← remove this dead code + + projects.append(account_models.Project(...)) + +# After (docstring addition) +""" +coact.Repo → IRI.Project mapping. +coact-api enforces membership via user impersonation (coactimp header); +repos returned are already scoped to the requesting user. +""" +``` + +## Operational Prerequisite + +The service account (`COACT_SERVICE_USER`) must be registered as a **bot user** (`isbot: true`) or **admin** in coact-api's `users` collection for `coactimp` to be honoured. This is an infrastructure/configuration concern, not a code change in iri-api. Without it, the `coactimp` header is rejected and coact-api returns repos for the service account identity instead of the target user. + +## Summary of Files to Change + +| File | Change | +|------|--------| +| `app/s3df/clients/coact.py` | `_get_client()`: add `coactimp` header in basic-auth branch | +| `app/s3df/clients/coact.py` | `get_user_repos()`: delegate to `get_my_repos(username)` | +| `app/s3df/account_adapter.py` | `get_projects()`: remove dead commented-out membership check; update docstring | diff --git a/design-docs/debug-facility-nersc.md b/design-docs/debug-facility-nersc.md new file mode 100644 index 00000000..01980615 --- /dev/null +++ b/design-docs/debug-facility-nersc.md @@ -0,0 +1,149 @@ +# Debug Report: `/facility` Endpoints Returning "NERSC" Data + +**Date:** 2026-05-21 +**Branch:** `feat/user-lookup-integration` + +--- + +## Summary + +Even with `IRI_API_ADAPTER_facility` correctly set to `S3DFFacilityAdapter`, NERSC references leak into `/facility` responses through **three separate mechanisms**: the `API_URL_ROOT` fallback, the OpenAPI schema, and the FastAPI server definition. The S3DF adapter itself returns clean SLAC data, but the surrounding framework still defaults to NERSC. + +--- + +## Hardcoded NERSC Couplings + +### 1. `API_URL_ROOT` fallback poisons all URI fields + +**File:** `app/config.py:39` +```python +API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") +``` + +**Impact chain:** +- `app/request_context.py:30` — `get_url_prefix()` returns `API_URL_ROOT` when no `x-forwarded-host` header is present: + ```python + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}" + # → "https://api.iri.nersc.gov/api/v1" + ``` +- `app/types/base.py:61` — every `NamedObject.self_uri` computed field uses `get_url_prefix()`: + ```python + return f"{get_url_prefix()}{self._self_path()}" + # → "https://api.iri.nersc.gov/api/v1/facility" + ``` +- `app/routers/facility/models.py:27` — `Site.resource_uris` uses `get_url_prefix()` +- `app/routers/facility/models.py:54` — `Facility.site_uris` uses `get_url_prefix()` + +**Result:** If `API_URL_ROOT` env var is not set (e.g., local dev without `local.env`, or a deployment missing this var), every `self_uri`, `site_uris`, and `resource_uris` field in the response contains `nersc.gov`. + +--- + +### 2. FastAPI `servers` definition hardcodes NERSC + +**File:** `app/main.py:51` +```python +APP = FastAPI(servers=[{"url": config.API_URL_ROOT}], **config.API_CONFIG) +``` + +The OpenAPI `servers` array (visible at `/openapi.json` and the Swagger "Try it out" dropdown) defaults to `https://api.iri.nersc.gov`. Any auto-generated client or Swagger UI will target NERSC by default. + +--- + +### 3. Pydantic `example` values reference NERSC/LBL + +**File:** `app/routers/facility/models.py` + +| Line | Field | Example Value | +|------|-------|---------------| +| 13 | `Site.short_name` | `"NERSC"` | +| 14 | `Site.operating_organization` | `"Lawrence Berkeley National Laboratory"` | +| 16 | `Site.locality_name` | `"Berkeley"` | +| 17 | `Site.state_or_province_name` | `"California"` | +| 18 | `Site.street_address` | `"1 Cyclotron Rd"` | +| 22 | `Site.latitude` | `37.8762` (Berkeley, CA) | +| 23 | `Site.longitude` | `-122.2506` (Berkeley, CA) | + +These appear in: OpenAPI spec, Swagger UI examples, and any client SDK generated from the schema. + +--- + +### 4. OpenTelemetry resource attributes + +**File:** `app/main.py:35` +```python +resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT # ← nersc.gov +}) +``` + +Traces exported to any collector will carry NERSC's endpoint in `service.endpoint`. + +--- + +### 5. `S3DFFacilityAdapter.self_uri` hardcoded (minor) + +**File:** `app/s3df/facility_adapter.py:47` +```python +self_uri="https://s3df-dev.slac.stanford.edu/api/v1/facility" +``` + +This is a Pydantic `extra` field passed at construction time. However, it's **overridden** by the computed `self_uri` property on `NamedObject` (which calls `get_url_prefix()`). So the value set here is effectively dead code — the serialized response uses the computed value. But it adds confusion during debugging. + +--- + +### 6. README and tools reference NERSC + +| File | Line | Content | +|------|------|---------| +| `README.md` | 6-8 | NERSC instance URLs in docs | +| `tools/globus.py` | 55 | `"NERSC-linked Globus identity"` | +| `test/test_filesystem.py` | 17 | Commented-out NERSC base URL | + +--- + +## Why the Problem Manifests + +The `dev-s3df` Makefile target (line 42-46) sets `API_URL_ROOT='http://127.0.0.1:8000'` but does **NOT** set `IRI_API_ADAPTER_facility`. The Dockerfile (line 10-11) sets `IRI_API_ADAPTER_account` and `IRI_SHOW_MISSING_ROUTES=true` but also does **NOT** set `API_URL_ROOT`. + +So in a deployed container: +1. `IRI_API_ADAPTER_facility` is set externally → S3DF adapter loads ✓ +2. `API_URL_ROOT` is **not** set → defaults to `https://api.iri.nersc.gov` ✗ +3. No `x-forwarded-host` header (direct access, no gateway) → `get_url_prefix()` falls back to `API_URL_ROOT` ✗ +4. All computed URIs in the response contain `nersc.gov` ✗ + +--- + +## All Affected Files + +| File | Line(s) | Coupling Type | +|------|---------|---------------| +| `app/config.py` | 39 | `API_URL_ROOT` default | +| `app/main.py` | 51 | FastAPI `servers` array | +| `app/main.py` | 35 | OTel `service.endpoint` | +| `app/request_context.py` | 30 | `get_url_prefix()` fallback | +| `app/routers/facility/models.py` | 13-23 | Pydantic examples (NERSC/Berkeley) | +| `app/s3df/facility_adapter.py` | 47 | Dead `self_uri` param | +| `tools/globus.py` | 55 | NERSC Globus instruction | +| `README.md` | 6-8 | NERSC instance docs | +| `Dockerfile` | (missing) | No `API_URL_ROOT` override | + +--- + +## Recommended Fixes + +1. **`app/config.py:39`** — Change default to a neutral value: + ```python + API_URL_ROOT = os.environ.get("API_URL_ROOT", "http://localhost:8000") + ``` + +2. **`Dockerfile`** — Add `API_URL_ROOT` env var (set to the actual S3DF deployment URL). + +3. **`app/routers/facility/models.py`** — Replace NERSC/Berkeley examples with generic or S3DF-relevant examples. + +4. **`app/s3df/facility_adapter.py:47`** — Remove the dead `self_uri` kwarg (it's overridden by the computed property). + +5. **`app/main.py:51`** — The `servers` list is already driven by `config.API_URL_ROOT`, so fixing #1 fixes this. + +6. **`tools/globus.py:55`** — Generalize the Globus identity instruction or make it configurable. diff --git a/design-docs/s3df-status-adapter.md b/design-docs/s3df-status-adapter.md new file mode 100644 index 00000000..90a71d41 --- /dev/null +++ b/design-docs/s3df-status-adapter.md @@ -0,0 +1,329 @@ +# S3DF Status Adapter — Technical Design Report + +## 1. Summary + +This document describes the implementation plan for `app/s3df/status_adapter.py` — the +S3DF-specific adapter that fulfills the IRI status API contract. Today the status router +(`/status`) has no S3DF implementation; requests require `IRI_API_ADAPTER_status` be set +to the demo adapter. The goal is a production adapter that queries real S3DF monitoring +infrastructure (Prometheus, InfluxDB) and maps results into the IRI status model +(Resources, Events, Incidents). + +The `status-pusher` CLI (separate repo) already queries these same backends and evaluates +health criteria — its query and evaluation logic serves as a reference for the data-plane +portion of this adapter. + +--- + +## 2. Architecture Context + +``` + ┌──────────────────────────────────────────────────────────────────────────┐ + │ IRI Facility API (FastAPI) │ + │ │ + │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ + │ │ /facility │ │ /account │ │ /compute │ │ /status │ │ + │ │ adapter ✓ │ │ adapter ✓ │ │ adapter ✓ │ │ adapter ✗ │ │ + │ └──────────────┘ └──────────────┘ └──────────────┘ └─────┬──────┘ │ + │ │ │ + └───────────────────────────────────────────────────────────────┼─────────┘ + │ + ┌────────────────────────────────────┘ + │ implements + ▼ + ┌────────────────────────┐ + │ S3DFStatusAdapter │ + │ app/s3df/status_ │ + │ adapter.py │ + └───────────┬────────────┘ + │ + ┌────────────┼─────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌──────────────┐ ┌────────────┐ ┌────────────────┐ + │ Prometheus │ │ InfluxDB │ │ Static config │ + │ (live health)│ │ (telegraf) │ │ (resource defs)│ + └──────────────┘ └────────────┘ └────────────────┘ +``` + +The existing `status-pusher` operates *outside* IRI as a scheduled cron/k8s job that +writes results to a git repo consumed by the Fettle dashboard. This adapter *replaces* +that indirection for the IRI API by querying backends directly at request time (or via a +background polling cache). + +--- + +## 3. IRI Status API Contract + +The abstract `FacilityAdapter` (in `app/routers/status/facility_adapter.py`) requires: + +| Method | Returns | Purpose | +|------------------|--------------------------|-----------------------------------| +| `get_resources` | `list[Resource]` | Filterable list of tracked resources | +| `get_resource` | `Resource` | Single resource by id | +| `get_events` | `list[Event]` | Status-change events | +| `get_event` | `Event` | Single event by id | +| `get_incidents` | `list[Incident]` | Planned/unplanned incidents | +| `get_incident` | `Incident` | Single incident by id | + +### Key Models + +``` +Resource: + id, name, description, last_modified, site_id, group, resource_type, + current_status (up|down|degraded|unknown), capability_ids + +Event: + id, name, description, last_modified, occurred_at, status, resource_id, + incident_id (optional) + +Incident: + id, name, description, last_modified, status, start, end, type + (planned|unplanned|reservation), resolution + (unresolved|cancelled|completed|extended|pending), resource_ids, event_ids +``` + +--- + +## 4. Reference: status-pusher + +`status-pusher` is a Click CLI that: + +1. Queries a metrics backend (Prometheus or InfluxDB). +2. Applies a success-condition comparator (eq, lt, gt, etc.) against the returned value. +3. Writes a timestamped CSV line (`zulu_ts, status_string, value`) to a log file in a git + repo (`slaclab/s3df-status`). +4. Commits + pushes the change (consumed by Fettle dashboard). + +### Relevant queries (from Makefile live tests): + +| Backend | Query | Meaning | +|------------|-------------------------------------------------------------------------------------------|------------------------------| +| Prometheus | `avg(avg_over_time(nmap_port_state{service='ssh',group='s3df'}[5m]))` | SSH gateway reachability | +| InfluxDB | `SELECT mean("status_code") FROM "monit_process" WHERE ("service" = 'slurmctld' OR "service" = 'slurmdbd') AND time > now()-5m GROUP BY "service"` | Slurm daemon health | + +### Data sources at S3DF: + +- **Prometheus**: `https://prometheus.slac.stanford.edu` — live probe metrics (nmap, etc.) +- **InfluxDB**: `https://influxdb.slac.stanford.edu`, database `telegraf` — system/service + metrics via Telegraf agents (monit, slurm, storage). + +--- + +## 5. Proposed Design + +### 5.1 Resource Registry (Static Config) + +Resources are *configuration*, not dynamic discovery. We define them via a YAML/dict +config that maps S3DF infrastructure to IRI `Resource` objects. Example: + +```yaml +resources: + - id: "s3df-ssh-gateway" + name: "SSH Login Gateway" + group: "access" + resource_type: "service" + capability_ids: [] + health_check: + backend: prometheus + query: "avg(avg_over_time(nmap_port_state{service='ssh',group='s3df'}[5m]))" + condition: {op: "eq", value: 1.0} + + - id: "s3df-slurmctld" + name: "Slurm Controller" + group: "compute" + resource_type: "compute" + capability_ids: [] + health_check: + backend: influxdb + db_name: "telegraf" + query: "SELECT mean(\"status_code\") FROM \"monit_process\" WHERE \"service\" = 'slurmctld' AND time > now()-5m" + condition: {op: "eq", value: 1.0} +``` + +This maps directly to the pattern `status-pusher` uses: query + comparator → up/down. + +### 5.2 Health Evaluation (Reused from status-pusher) + +``` + ┌──────────────────────────────────────────────────────────────┐ + │ HealthChecker (internal module) │ + │ │ + │ prometheus_query(url, query) -> (epoch_ts, value) │ + │ influx_query(url, db, query) -> (epoch_ts, value) │ + │ evaluate(value, condition) -> Status (up|down|degraded) │ + └──────────────────────────────────────────────────────────────┘ +``` + +This directly reuses the logic from `status_pusher.py`: +- `prometheus_query()` → wraps `PrometheusConnect.custom_query()` +- `influx_query()` → HTTP GET to InfluxDB `/query` endpoint +- `evaluate()` → applies comparator (eq/lt/gt/gte/lte) to determine status + +### 5.3 Caching / Polling Strategy + +Querying backends on every HTTP request would be expensive and fragile. Two options: + +**Option A: Background Poller (Recommended)** +``` + ┌─────────────────────────────────────────────────────────────┐ + │ BackgroundPoller (asyncio.Task, runs at startup) │ + │ │ + │ Every N seconds (configurable, default 60): │ + │ for resource in config.resources: │ + │ result = health_check(resource) │ + │ cache[resource.id] = StatusSnapshot(ts, status, value) │ + │ if status_changed: │ + │ events.append(new Event) │ + └─────────────────────────────────────────────────────────────┘ +``` +- Cache is an in-memory dict protected by asyncio Lock +- Events are accumulated in-memory (optionally persisted to SQLite/file for restart) +- `get_resources()` reads from cache +- `current_status` on each Resource reflects the latest polled value + +**Option B: On-demand with TTL Cache** +- Simpler; queries backend on first request, caches for TTL seconds +- Risk: slow/failed backend blocks API response + +### 5.4 Events & Incidents + +Events are generated when status *transitions* occur (e.g., up→down). The poller detects +transitions by comparing the new status to the cached previous status. + +``` + Time ──────────────────────────────────────────────► + + Resource: s3df-ssh-gateway + Poll 1: status=up (no event) + Poll 2: status=up (no event) + Poll 3: status=down → Event(occurred_at=now, status=down) + → Incident(type=unplanned, start=now, status=down) + Poll 4: status=down (no event, incident still open) + Poll 5: status=up → Event(occurred_at=now, status=up) + → Incident.end=now, resolution=completed +``` + +Incidents are auto-created for unplanned outages. Planned incidents can be injected via: +- Future: a maintenance-schedule config or API +- For MVP: all incidents are `unplanned` and auto-generated + +### 5.5 Module Structure + +``` +app/s3df/ +├── status_adapter.py # S3DFStatusAdapter class +├── status/ +│ ├── __init__.py +│ ├── config.py # Resource registry + env-driven settings +│ ├── health_checker.py # Prometheus/InfluxDB query + evaluation +│ ├── poller.py # Background polling loop +│ └── store.py # In-memory event/incident store +└── ... +``` + +### 5.6 Data Flow (Request Path) + +``` + Client GET /status/resources + │ + ▼ + status.py router + │ + ▼ + S3DFStatusAdapter.get_resources(filters...) + │ + ├─► reads resource list from config (static) + ├─► enriches current_status from cache (poller-written) + ├─► applies filters (name, group, type, status, modified_since, etc.) + └─► returns paginated list +``` + +``` + Client GET /status/events?resource_id=X&from=T1&to=T2 + │ + ▼ + S3DFStatusAdapter.get_events(filters...) + │ + ├─► reads event log from in-memory store + ├─► applies filters (resource_id, status, time range, etc.) + └─► returns paginated list +``` + +--- + +## 6. Configuration (Environment Variables) + +| Variable | Purpose | Default | +|----------------------------|--------------------------------------------|------------------------------------| +| `S3DF_PROMETHEUS_URL` | Prometheus endpoint | `https://prometheus.slac.stanford.edu` | +| `S3DF_INFLUXDB_URL` | InfluxDB endpoint | `https://influxdb.slac.stanford.edu` | +| `S3DF_INFLUXDB_DB` | InfluxDB database name | `telegraf` | +| `S3DF_STATUS_POLL_INTERVAL`| Seconds between health checks | `60` | +| `S3DF_STATUS_CONFIG_PATH` | Path to resource registry YAML (optional) | built-in defaults | + +--- + +## 7. Comparison: status-pusher vs Status Adapter + +| Aspect | status-pusher | S3DF Status Adapter | +|---------------------|----------------------------------|----------------------------------------| +| Runtime model | Cron job / k8s CronJob | Long-running FastAPI background task | +| Output | CSV in git repo → Fettle | In-memory cache → IRI REST API | +| Query backends | Prometheus, InfluxDB | Same (reuse query logic) | +| Evaluation logic | comparator(value, threshold) | Same (reuse evaluation logic) | +| Event tracking | None (single point-in-time) | Stateful: detects transitions | +| Incident tracking | None | Auto-created on status transitions | +| Auth | N/A (runs internally) | Unauthenticated (status is public) | + +--- + +## 8. Implementation Steps + +1. **Create `app/s3df/status/health_checker.py`** — Extract and adapt + `prometheus_query()` and `influx_query()` from `status-pusher`, adding async support + (use `httpx` instead of `requests` for non-blocking I/O). Port the condition evaluator. + +2. **Create `app/s3df/status/config.py`** — Define the resource registry as a Python + dict/dataclass (with optional YAML override). Include health check definitions per + resource. + +3. **Create `app/s3df/status/store.py`** — In-memory store for Events and Incidents with + append-only log semantics. Provides filtered query methods matching the adapter + interface. + +4. **Create `app/s3df/status/poller.py`** — Background asyncio task that periodically + runs health checks, updates resource status, and emits Events/Incidents on transitions. + +5. **Create `app/s3df/status_adapter.py`** — `S3DFStatusAdapter` class implementing + `app.routers.status.facility_adapter.FacilityAdapter`. Wires together config, cache, + and store. Starts poller on first access or app startup. + +6. **Register** — Set `IRI_API_ADAPTER_status=app.s3df.status_adapter.S3DFStatusAdapter` + in Dockerfile/deployment config. + +7. **Tests** — Unit tests mirroring `status-pusher/test/` patterns: mock Prometheus and + InfluxDB responses, verify status evaluation, event generation, and adapter filtering. + +--- + +## 9. Open Questions + +- **Persistence across restarts**: Should events/incidents survive process restarts? + Options: SQLite file, Redis, or accept ephemeral (events regenerate from live state). +- **Planned maintenance**: How should planned incidents be ingested? Config file, external + API call, or manual injection? +- **Fettle coexistence**: Should this adapter *replace* the status-pusher→Fettle pipeline + or run alongside it? (IRI could become the single source of truth for both.) +- **Additional resources**: What is the full set of resources to monitor? (SSH gateway, + slurmctld, slurmdbd, storage, network, web services, etc.) +- **Degraded vs Down thresholds**: Should we support a third threshold for `degraded` + status (e.g., partial failure), or only binary up/down per status-pusher's model? + +--- + +## 10. Dependencies + +- `prometheus-api-client` (already used by status-pusher) +- `httpx` (async HTTP for InfluxDB queries — preferred over sync `requests` in async app) +- No new external services required; uses existing Prometheus + InfluxDB infrastructure From 73225649730939fd318ce5dc7177896c6e329164 Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Mon, 1 Jun 2026 12:42:55 -0700 Subject: [PATCH 3/6] restructuring status adapter into a module --- app/s3df/status/__init__.py | 38 ++ app/s3df/status/config.py | 211 ++++++++ app/s3df/status/health_checker.py | 102 ++++ app/s3df/status/poller.py | 79 +++ app/s3df/status/store.py | 147 ++++++ app/s3df/status_adapter.py | 487 +----------------- .../s3df-status-adapter-alternatives.md | 250 +++++++++ design-docs/s3df-status-adapter.md | 257 +++++++++ 8 files changed, 1109 insertions(+), 462 deletions(-) create mode 100644 app/s3df/status/__init__.py create mode 100644 app/s3df/status/config.py create mode 100644 app/s3df/status/health_checker.py create mode 100644 app/s3df/status/poller.py create mode 100644 app/s3df/status/store.py create mode 100644 design-docs/s3df-status-adapter-alternatives.md diff --git a/app/s3df/status/__init__.py b/app/s3df/status/__init__.py new file mode 100644 index 00000000..8ffc082a --- /dev/null +++ b/app/s3df/status/__init__.py @@ -0,0 +1,38 @@ +""" +S3DF status adapter package. + +Splits the adapter into focused modules: + + * ``config`` — health-check model, resource registry, settings + * ``health_checker`` — Prometheus/InfluxDB query + status evaluation + * ``store`` — in-memory current-status / event / incident store + * ``poller`` — background polling loop + +The public ``S3DFStatusAdapter`` lives in ``app.s3df.status_adapter``. +""" + +from .config import ( + REGISTRY, + Backend, + Condition, + HealthCheck, + MonitoredResource, + StatusSettings, +) +from .health_checker import HealthChecker, HealthResult, evaluate +from .poller import StatusPoller +from .store import StatusStore + +__all__ = [ + "REGISTRY", + "Backend", + "Condition", + "HealthCheck", + "MonitoredResource", + "StatusSettings", + "HealthChecker", + "HealthResult", + "evaluate", + "StatusPoller", + "StatusStore", +] diff --git a/app/s3df/status/config.py b/app/s3df/status/config.py new file mode 100644 index 00000000..bf29c94a --- /dev/null +++ b/app/s3df/status/config.py @@ -0,0 +1,211 @@ +""" +Configuration for the S3DF status adapter. + +Holds the static, request-independent pieces of the adapter: + + * The health-check model (``Backend``, ``Condition``, ``HealthCheck``) — a + declarative success-criterion mirroring the model ``status-pusher`` uses. + * ``MonitoredResource`` — pairs an IRI ``Resource`` template with the health + check that drives it (see the duplication note below). + * ``REGISTRY`` — the set of S3DF resources surfaced by ``/status``. + * ``StatusSettings`` — environment-driven configuration. + +Avoiding Resource/ResourceDef duplication +------------------------------------------ +A resource's stable identity/metadata (id, name, description, group, +resource_type, capability_ids) is declared exactly once, as an IRI +``Resource`` template inside ``MonitoredResource``. The runtime-owned fields +(``current_status``, ``last_modified``, ``site_id``) are placeholders here and +are overlaid by the store when it projects a live view, so the fields are never +re-listed in a parallel definition. + +Required/optional env vars: + S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) + S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) + S3DF_INFLUXDB_DB InfluxDB database (default telegraf) + S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) + S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) + S3DF_SITE_ID Site id for resources (default s3df) + S3DF_STATUS_TLS_VERIFY true | false | (default false) +""" + +import datetime +import operator +import os +from dataclasses import dataclass +from enum import Enum +from typing import Callable + +from app.routers.status.models import Resource, ResourceType, Status + + +def utc_now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +# Placeholder timestamp baked into static Resource templates. The store is the +# authority for ``last_modified`` and overwrites this when projecting a view. +_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + + +class Backend(str, Enum): + """Metrics backend a health check queries.""" + + prometheus = "prometheus" + influxdb = "influxdb" + + +_COMPARATORS: dict[str, Callable[[float, float], bool]] = { + "eq": operator.eq, + "ne": operator.ne, + "lt": operator.lt, + "lte": operator.le, + "gt": operator.gt, + "gte": operator.ge, +} + + +@dataclass(frozen=True) +class Condition: + """A comparison of an observed metric value against a threshold.""" + + comparator: str + value: float + + def met(self, observed: float) -> bool: + op = _COMPARATORS.get(self.comparator) + if op is None: + raise ValueError(f"Unknown comparator: {self.comparator}") + return bool(op(observed, self.value)) + + +@dataclass(frozen=True) +class HealthCheck: + """How to determine a resource's status from a metrics backend.""" + + backend: Backend + query: str + up_when: Condition + db_name: str | None = None # InfluxDB only + degraded_when: Condition | None = None + + +@dataclass(frozen=True) +class MonitoredResource: + """Pairs a static ``Resource`` template with the health check driving it. + + The ``Resource`` carries the resource's stable identity/metadata exactly + once. Its dynamic fields (``current_status``, ``last_modified``, + ``site_id``) are placeholders that the store overlays when it builds the + live view, which is what lets us avoid a separate ``ResourceDef`` that would + otherwise re-list the same fields. + """ + + resource: Resource + health_check: HealthCheck + + +def _template( + *, + id: str, + name: str, + description: str, + group: str, + resource_type: ResourceType, + capability_ids: tuple[str, ...] = (), +) -> Resource: + """Build a static Resource template. Dynamic fields are placeholders the + store overwrites (``current_status``, ``last_modified``, ``site_id``).""" + return Resource( + id=id, + name=name, + description=description, + last_modified=_EPOCH, + site_id="", # overlaid by the store from settings.site_id + group=group, + resource_type=resource_type, + current_status=Status.unknown, + capability_ids=list(capability_ids), + ) + + +# The set of S3DF resources surfaced by /status. Queries mirror those exercised +# by status-pusher's live tests (see status-pusher/Makefile). Extend as needed. +REGISTRY: list[MonitoredResource] = [ + MonitoredResource( + resource=_template( + id="s3df-ssh-gateway", + name="SSH Login Gateway", + description="S3DF interactive SSH login gateway reachability.", + group="access", + resource_type=ResourceType.service, + ), + health_check=HealthCheck( + backend=Backend.prometheus, + query="avg( avg_over_time(nmap_port_state{service=`ssh`,group=`s3df`}[5m]) )", + up_when=Condition("eq", 1.0), + ), + ), + MonitoredResource( + resource=_template( + id="s3df-slurmctld", + name="Slurm Controller (slurmctld)", + description="Slurm workload manager controller daemon health.", + group="compute", + resource_type=ResourceType.compute, + ), + health_check=HealthCheck( + backend=Backend.influxdb, + db_name="telegraf", + query=( + 'SELECT mean("status_code") FROM "monit_process" ' + "WHERE \"service\" = 'slurmctld' AND time > now()-5m" + ), + up_when=Condition("eq", 1.0), + ), + ), + MonitoredResource( + resource=_template( + id="s3df-slurmdbd", + name="Slurm DB Daemon (slurmdbd)", + description="Slurm accounting database daemon health.", + group="compute", + resource_type=ResourceType.compute, + ), + health_check=HealthCheck( + backend=Backend.influxdb, + db_name="telegraf", + query=( + 'SELECT mean("status_code") FROM "monit_process" ' + "WHERE \"service\" = 'slurmdbd' AND time > now()-5m" + ), + up_when=Condition("eq", 1.0), + ), + ), +] + + +class StatusSettings: + """Environment-driven settings for the S3DF status adapter.""" + + def __init__(self) -> None: + self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") + self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") + self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") + self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) + self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) + # NOTE: the /facility adapter currently mints a random site uuid per + # process, so this id is not a guaranteed cross-reference yet. Set + # S3DF_SITE_ID once a stable site identifier is established. + self.site_id = os.getenv("S3DF_SITE_ID", "s3df") + self.tls_verify = self._parse_verify(os.getenv("S3DF_STATUS_TLS_VERIFY", "false")) + + @staticmethod + def _parse_verify(raw: str) -> bool | str: + low = raw.strip().lower() + if low in ("true", "1", "yes", "on"): + return True + if low in ("false", "0", "no", "off", ""): + return False + # Anything else is treated as a path to a CA bundle. + return raw diff --git a/app/s3df/status/health_checker.py b/app/s3df/status/health_checker.py new file mode 100644 index 00000000..7d68020f --- /dev/null +++ b/app/s3df/status/health_checker.py @@ -0,0 +1,102 @@ +""" +Health checker for the S3DF status adapter. + +Runs a :class:`~app.s3df.status.config.HealthCheck` against its metrics backend +(Prometheus or InfluxDB) over ``httpx`` and reduces the response to a single +:class:`~app.routers.status.models.Status` via :func:`evaluate`. +""" + +import datetime +import logging +from dataclasses import dataclass + +import httpx + +from app.routers.status.models import Status + +from .config import Backend, HealthCheck, StatusSettings, utc_now + +logger = logging.getLogger(__name__) + + +@dataclass +class HealthResult: + """Outcome of a single health check.""" + + status: Status + value: float | None + observed_at: datetime.datetime + error: str | None = None + + +def evaluate(check: HealthCheck, value: float | None) -> Status: + """Map an observed metric value to a Status using the check's conditions.""" + if value is None: + return Status.unknown + if check.up_when.met(value): + return Status.up + if check.degraded_when is not None and check.degraded_when.met(value): + return Status.degraded + return Status.down + + +class HealthChecker: + """Runs a HealthCheck against its backend and evaluates the result. + + The caller owns the ``httpx.AsyncClient`` lifecycle (created from within the + running event loop) and passes it in. + """ + + def __init__(self, settings: StatusSettings, client: httpx.AsyncClient): + self.settings = settings + self.client = client + + async def check(self, check: HealthCheck) -> HealthResult: + now = utc_now() + try: + if check.backend == Backend.prometheus: + value = await self._prometheus_query(check.query) + elif check.backend == Backend.influxdb: + db = check.db_name or self.settings.influxdb_db + value = await self._influx_query(db, check.query) + else: # pragma: no cover - guarded by enum + return HealthResult(Status.unknown, None, now, f"unknown backend {check.backend}") + except Exception as exc: # noqa: BLE001 - any failure -> unknown + logger.warning("Health check failed (%s): %s", check.backend.value, exc) + return HealthResult(Status.unknown, None, now, str(exc)) + return HealthResult(evaluate(check, value), value, now) + + async def _prometheus_query(self, query: str) -> float | None: + """Instant query via the Prometheus HTTP API; returns the scalar value.""" + url = self.settings.prometheus_url.rstrip("/") + "/api/v1/query" + resp = await self.client.get(url, params={"query": query}, timeout=self.settings.http_timeout) + resp.raise_for_status() + data = resp.json() + if data.get("status") != "success": + raise RuntimeError(f"prometheus error: {data.get('error', 'unknown')}") + result = data.get("data", {}).get("result", []) + if not result: + return None + # Instant vector sample: value == [epoch_ts, "stringified_value"] + return float(result[0]["value"][1]) + + async def _influx_query(self, db_name: str, query: str) -> float | None: + """Query the InfluxDB HTTP API; returns the most recent scalar value.""" + url = self.settings.influxdb_url.rstrip("/") + "/query" + resp = await self.client.get( + url, params={"q": query, "db": db_name}, timeout=self.settings.http_timeout + ) + resp.raise_for_status() + data = resp.json() + results = data.get("results", []) + if not results: + return None + series = results[0].get("series") + if not series: + return None + values = series[0].get("values") + if not values: + return None + # Each row is [time, value]; take the latest row's value column. + row = values[-1] + return float(row[1]) if row[1] is not None else None diff --git a/app/s3df/status/poller.py b/app/s3df/status/poller.py new file mode 100644 index 00000000..56584cf0 --- /dev/null +++ b/app/s3df/status/poller.py @@ -0,0 +1,79 @@ +""" +Background poller for the S3DF status adapter. + +Periodically runs the health check for each monitored resource and feeds the +results into the :class:`~app.s3df.status.store.StatusStore`. The ``httpx`` +client is created lazily inside the running event loop (see :meth:`start`). +""" + +import asyncio +import logging + +import httpx + +from .config import MonitoredResource, StatusSettings +from .health_checker import HealthChecker +from .store import StatusStore + +logger = logging.getLogger(__name__) + + +class StatusPoller: + """Periodically runs health checks and feeds results into the store.""" + + def __init__(self, store: StatusStore, settings: StatusSettings, monitored: list[MonitoredResource]): + self.store = store + self.settings = settings + self.monitored = monitored + self._client: httpx.AsyncClient | None = None + self._checker: HealthChecker | None = None + self._task: asyncio.Task | None = None + + async def start(self) -> None: + """Create the HTTP client (inside the running loop), run one initial poll, + then launch the periodic background loop.""" + self._client = httpx.AsyncClient(verify=self.settings.tls_verify) + self._checker = HealthChecker(self.settings, self._client) + await self._poll_once() # bounded by per-query timeouts; never raises + self._task = asyncio.create_task(self._run(), name="s3df-status-poller") + logger.info( + "S3DF status poller started (%d resources, interval=%ss)", + len(self.monitored), + self.settings.poll_interval, + ) + + async def _run(self) -> None: + try: + while True: + await asyncio.sleep(self.settings.poll_interval) + await self._poll_once() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - keep the server alive if the loop dies + logger.exception("S3DF status poller loop crashed") + + async def _poll_once(self) -> None: + assert self._checker is not None + monitored = self.monitored + results = await asyncio.gather( + *(self._checker.check(m.health_check) for m in monitored), + return_exceptions=True, + ) + for m, res in zip(monitored, results): + if isinstance(res, Exception): + logger.warning("Health check raised for %s: %s", m.resource.id, res) + continue + self.store.record(m.resource.id, res) + + async def aclose(self) -> None: + """Cancel the loop and close the HTTP client. For tests/lifespan wiring.""" + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + if self._client is not None: + await self._client.aclose() + self._client = None diff --git a/app/s3df/status/store.py b/app/s3df/status/store.py new file mode 100644 index 00000000..398d5663 --- /dev/null +++ b/app/s3df/status/store.py @@ -0,0 +1,147 @@ +""" +In-memory store for the S3DF status adapter. + +Holds the current status per resource plus an event log and incident set, and +turns poll results into IRI ``Event`` / ``Incident`` objects on status +transitions. State is per-process and volatile (lost on restart, diverges +across uvicorn workers). +""" + +import datetime +import uuid + +from app.routers.status.models import ( + Event, + Incident, + IncidentType, + Resolution, + Resource, + Status, +) + +from .config import utc_now +from .health_checker import HealthResult + + +class StatusStore: + """Holds current status per resource plus an event log and incident set. + + Single-writer model: only the poller calls ``record()``, and ``record()`` has + no internal ``await`` points, so each mutation is atomic with respect to the + asyncio event loop. Reader methods return freshly built/copied lists, so a + request handler never observes a half-applied update. State is per-process and + volatile. + + The store is given the static ``Resource`` templates and is the authority for + their dynamic fields: it overlays ``site_id``, ``current_status`` and + ``last_modified`` when projecting a live view in :meth:`resources`. + """ + + def __init__(self, site_id: str, resources: list[Resource]): + self.site_id = site_id + self._templates: dict[str, Resource] = {r.id: r for r in resources} + self._created_at = utc_now() + self._status: dict[str, Status] = {} + self._last_modified: dict[str, datetime.datetime] = {} + self._events: list[Event] = [] + self._incidents: dict[str, Incident] = {} + self._open_incident: dict[str, str] = {} # resource_id -> incident_id + + # -- writer ------------------------------------------------------------ + + def record(self, resource_id: str, result: HealthResult) -> None: + """Record a poll result, emitting an Event + Incident change on transition.""" + if resource_id not in self._templates: + return + prev = self._status.get(resource_id) + new = result.status + ts = result.observed_at + + if prev == new: + return # steady state — nothing to record + + baseline = prev is None + self._status[resource_id] = new + self._last_modified[resource_id] = ts + + event = self._make_event(resource_id, new, ts, baseline) + + if new in (Status.down, Status.degraded): + inc_id = self._open_incident.get(resource_id) + if inc_id is None: + incident = self._make_incident(resource_id, new, ts) + self._incidents[incident.id] = incident + self._open_incident[resource_id] = incident.id + inc_id = incident.id + else: + incident = self._incidents[inc_id] + incident.status = new + incident.last_modified = ts + event.incident_id = inc_id + self._incidents[inc_id].event_ids.append(event.id) + elif new == Status.up: + inc_id = self._open_incident.pop(resource_id, None) + if inc_id is not None: + incident = self._incidents[inc_id] + incident.status = Status.up + incident.end = ts + incident.resolution = Resolution.completed + incident.last_modified = ts + event.incident_id = inc_id + incident.event_ids.append(event.id) + # Status.unknown: record the event but never open/close incidents — + # a monitoring-backend failure is not a confirmed resource outage. + + self._events.append(event) + + def _make_event(self, resource_id: str, status: Status, ts: datetime.datetime, baseline: bool) -> Event: + rname = self._templates[resource_id].name + verb = "initial status" if baseline else "status changed to" + return Event( + id=str(uuid.uuid4()), + name=f"{rname}: {status.value}", + description=f"{rname} {verb} {status.value}.", + last_modified=ts, + occurred_at=ts, + status=status, + resource_id=resource_id, + ) + + def _make_incident(self, resource_id: str, status: Status, ts: datetime.datetime) -> Incident: + rname = self._templates[resource_id].name + return Incident( + id=str(uuid.uuid4()), + name=f"{rname} {status.value}", + description=f"Automatically opened: {rname} observed {status.value}.", + last_modified=ts, + status=status, + start=ts, + type=IncidentType.unplanned, + resolution=Resolution.unresolved, + resource_ids=[resource_id], + event_ids=[], + ) + + # -- readers ----------------------------------------------------------- + + def resources(self) -> list[Resource]: + """Project live Resource views by overlaying the runtime-owned fields + (site_id, current_status, last_modified) onto the static templates.""" + out: list[Resource] = [] + for id_, tmpl in self._templates.items(): + out.append( + tmpl.model_copy( + update={ + "site_id": self.site_id, + "current_status": self._status.get(id_, Status.unknown), + "last_modified": self._last_modified.get(id_, self._created_at), + } + ) + ) + return out + + def events(self) -> list[Event]: + return sorted(self._events, key=lambda e: (e.occurred_at, e.id)) + + def incidents(self) -> list[Incident]: + return sorted(self._incidents.values(), key=lambda i: (i.start, i.id)) diff --git a/app/s3df/status_adapter.py b/app/s3df/status_adapter.py index 92d222d2..650dd86a 100644 --- a/app/s3df/status_adapter.py +++ b/app/s3df/status_adapter.py @@ -7,483 +7,39 @@ Design (see design-docs/s3df-status-adapter.md): - * Resources are STATIC config (``RESOURCE_REGISTRY``). Each carries a health - check: a backend + query + condition (comparator, threshold) — mirroring the - success-criterion model used by ``status-pusher``. - * A background poller runs every ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries - each resource's backend, maps the result to a Status (up/down/degraded/ - unknown), and records it in an in-memory store. + * Resources are STATIC config (``app.s3df.status.config.REGISTRY``). Each + pairs an IRI ``Resource`` template with a health check (backend + query + + condition) — mirroring the success-criterion model used by ``status-pusher``. + * A background poller (``app.s3df.status.poller``) runs every + ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries each resource's backend, maps + the result to a Status (up/down/degraded/unknown), and records it in an + in-memory store (``app.s3df.status.store``). * The store detects status TRANSITIONS to emit Events and to open/close unplanned Incidents. +This module hosts only the ``FacilityAdapter`` implementation; the registry, +health checker, store and poller live in the ``app.s3df.status`` package. + Lifecycle note: IRI constructs adapters synchronously at import/router-build time — before the asyncio event loop exists — so the poller and its ``httpx`` client are created LAZILY on the first request (``_ensure_started``), where the loop is running. State is in-memory and per-process: it is lost on restart and diverges across multiple uvicorn workers. Persistence and FastAPI-lifespan shutdown wiring are tracked as follow-ups in the design doc. - -Required/optional env vars: - S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) - S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) - S3DF_INFLUXDB_DB InfluxDB database (default telegraf) - S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) - S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) - S3DF_SITE_ID Site id for resources (default s3df) - S3DF_STATUS_TLS_VERIFY true | false | (default false) """ import asyncio import datetime import logging -import operator -import os -import uuid -from dataclasses import dataclass -from enum import Enum -from typing import Callable - -import httpx from app.routers.status import facility_adapter as status_adapter from app.routers.status import models as status_models -from app.routers.status.models import ( - Event, - Incident, - IncidentType, - Resolution, - Resource, - ResourceType, - Status, -) - -logger = logging.getLogger(__name__) - - -def _utc_now() -> datetime.datetime: - return datetime.datetime.now(datetime.timezone.utc) - - -# --------------------------------------------------------------------------- -# Configuration / resource registry -# --------------------------------------------------------------------------- - - -class Backend(str, Enum): - """Metrics backend a health check queries.""" - - prometheus = "prometheus" - influxdb = "influxdb" - - -_COMPARATORS: dict[str, Callable[[float, float], bool]] = { - "eq": operator.eq, - "ne": operator.ne, - "lt": operator.lt, - "lte": operator.le, - "gt": operator.gt, - "gte": operator.ge, -} - - -@dataclass(frozen=True) -class Condition: - """A comparison of an observed metric value against a threshold.""" - - comparator: str - value: float - - def met(self, observed: float) -> bool: - op = _COMPARATORS.get(self.comparator) - if op is None: - raise ValueError(f"Unknown comparator: {self.comparator}") - return bool(op(observed, self.value)) - - -@dataclass(frozen=True) -class HealthCheck: - """How to determine a resource's status from a metrics backend.""" - - backend: Backend - query: str - up_when: Condition - db_name: str | None = None # InfluxDB only - degraded_when: Condition | None = None - - -@dataclass(frozen=True) -class ResourceDef: - """Static definition of a monitored resource + its health check.""" - - id: str - name: str - description: str - group: str - resource_type: ResourceType - health_check: HealthCheck - capability_ids: tuple[str, ...] = () - - -class StatusSettings: - """Environment-driven settings for the S3DF status adapter.""" - - def __init__(self) -> None: - self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") - self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") - self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") - self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) - self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) - # NOTE: the /facility adapter currently mints a random site uuid per - # process, so this id is not a guaranteed cross-reference yet. Set - # S3DF_SITE_ID once a stable site identifier is established. - self.site_id = os.getenv("S3DF_SITE_ID", "s3df") - self.tls_verify = self._parse_verify(os.getenv("S3DF_STATUS_TLS_VERIFY", "false")) - - @staticmethod - def _parse_verify(raw: str) -> bool | str: - low = raw.strip().lower() - if low in ("true", "1", "yes", "on"): - return True - if low in ("false", "0", "no", "off", ""): - return False - # Anything else is treated as a path to a CA bundle. - return raw - - -# The set of S3DF resources surfaced by /status. Queries mirror those exercised -# by status-pusher's live tests (see status-pusher/Makefile). Extend as needed. -RESOURCE_REGISTRY: list[ResourceDef] = [ - ResourceDef( - id="s3df-ssh-gateway", - name="SSH Login Gateway", - description="S3DF interactive SSH login gateway reachability.", - group="access", - resource_type=ResourceType.service, - health_check=HealthCheck( - backend=Backend.prometheus, - query="avg( avg_over_time(nmap_port_state{service=`ssh`,group=`s3df`}[5m]) )", - up_when=Condition("eq", 1.0), - ), - ), - ResourceDef( - id="s3df-slurmctld", - name="Slurm Controller (slurmctld)", - description="Slurm workload manager controller daemon health.", - group="compute", - resource_type=ResourceType.compute, - health_check=HealthCheck( - backend=Backend.influxdb, - db_name="telegraf", - query=( - 'SELECT mean("status_code") FROM "monit_process" ' - "WHERE \"service\" = 'slurmctld' AND time > now()-5m" - ), - up_when=Condition("eq", 1.0), - ), - ), - ResourceDef( - id="s3df-slurmdbd", - name="Slurm DB Daemon (slurmdbd)", - description="Slurm accounting database daemon health.", - group="compute", - resource_type=ResourceType.compute, - health_check=HealthCheck( - backend=Backend.influxdb, - db_name="telegraf", - query=( - 'SELECT mean("status_code") FROM "monit_process" ' - "WHERE \"service\" = 'slurmdbd' AND time > now()-5m" - ), - up_when=Condition("eq", 1.0), - ), - ), -] - - -# --------------------------------------------------------------------------- -# Health checker — async Prometheus / InfluxDB queries + evaluation -# --------------------------------------------------------------------------- - - -@dataclass -class HealthResult: - """Outcome of a single health check.""" - - status: Status - value: float | None - observed_at: datetime.datetime - error: str | None = None - - -def evaluate(check: HealthCheck, value: float | None) -> Status: - """Map an observed metric value to a Status using the check's conditions.""" - if value is None: - return Status.unknown - if check.up_when.met(value): - return Status.up - if check.degraded_when is not None and check.degraded_when.met(value): - return Status.degraded - return Status.down - - -class HealthChecker: - """Runs a HealthCheck against its backend and evaluates the result. - - The caller owns the ``httpx.AsyncClient`` lifecycle (created from within the - running event loop) and passes it in. - """ - - def __init__(self, settings: StatusSettings, client: httpx.AsyncClient): - self.settings = settings - self.client = client - - async def check(self, check: HealthCheck) -> HealthResult: - now = _utc_now() - try: - if check.backend == Backend.prometheus: - value = await self._prometheus_query(check.query) - elif check.backend == Backend.influxdb: - db = check.db_name or self.settings.influxdb_db - value = await self._influx_query(db, check.query) - else: # pragma: no cover - guarded by enum - return HealthResult(Status.unknown, None, now, f"unknown backend {check.backend}") - except Exception as exc: # noqa: BLE001 - any failure -> unknown - logger.warning("Health check failed (%s): %s", check.backend.value, exc) - return HealthResult(Status.unknown, None, now, str(exc)) - return HealthResult(evaluate(check, value), value, now) - - async def _prometheus_query(self, query: str) -> float | None: - """Instant query via the Prometheus HTTP API; returns the scalar value.""" - url = self.settings.prometheus_url.rstrip("/") + "/api/v1/query" - resp = await self.client.get(url, params={"query": query}, timeout=self.settings.http_timeout) - resp.raise_for_status() - data = resp.json() - if data.get("status") != "success": - raise RuntimeError(f"prometheus error: {data.get('error', 'unknown')}") - result = data.get("data", {}).get("result", []) - if not result: - return None - # Instant vector sample: value == [epoch_ts, "stringified_value"] - return float(result[0]["value"][1]) - async def _influx_query(self, db_name: str, query: str) -> float | None: - """Query the InfluxDB HTTP API; returns the most recent scalar value.""" - url = self.settings.influxdb_url.rstrip("/") + "/query" - resp = await self.client.get( - url, params={"q": query, "db": db_name}, timeout=self.settings.http_timeout - ) - resp.raise_for_status() - data = resp.json() - results = data.get("results", []) - if not results: - return None - series = results[0].get("series") - if not series: - return None - values = series[0].get("values") - if not values: - return None - # Each row is [time, value]; take the latest row's value column. - row = values[-1] - return float(row[1]) if row[1] is not None else None - - -# --------------------------------------------------------------------------- -# In-memory store — current status, events, incidents -# --------------------------------------------------------------------------- - - -class StatusStore: - """Holds current status per resource plus an event log and incident set. - - Single-writer model: only the poller calls ``record()``, and ``record()`` has - no internal ``await`` points, so each mutation is atomic with respect to the - asyncio event loop. Reader methods return freshly built/copied lists, so a - request handler never observes a half-applied update. State is per-process and - volatile. - """ - - def __init__(self, site_id: str, resource_defs: list[ResourceDef]): - self.site_id = site_id - self._defs: dict[str, ResourceDef] = {d.id: d for d in resource_defs} - self._created_at = _utc_now() - self._status: dict[str, Status] = {} - self._last_modified: dict[str, datetime.datetime] = {} - self._events: list[Event] = [] - self._incidents: dict[str, Incident] = {} - self._open_incident: dict[str, str] = {} # resource_id -> incident_id - - # -- writer ------------------------------------------------------------ - - def record(self, resource_id: str, result: HealthResult) -> None: - """Record a poll result, emitting an Event + Incident change on transition.""" - if resource_id not in self._defs: - return - prev = self._status.get(resource_id) - new = result.status - ts = result.observed_at - - if prev == new: - return # steady state — nothing to record - - baseline = prev is None - self._status[resource_id] = new - self._last_modified[resource_id] = ts - - event = self._make_event(resource_id, new, ts, baseline) - - if new in (Status.down, Status.degraded): - inc_id = self._open_incident.get(resource_id) - if inc_id is None: - incident = self._make_incident(resource_id, new, ts) - self._incidents[incident.id] = incident - self._open_incident[resource_id] = incident.id - inc_id = incident.id - else: - incident = self._incidents[inc_id] - incident.status = new - incident.last_modified = ts - event.incident_id = inc_id - self._incidents[inc_id].event_ids.append(event.id) - elif new == Status.up: - inc_id = self._open_incident.pop(resource_id, None) - if inc_id is not None: - incident = self._incidents[inc_id] - incident.status = Status.up - incident.end = ts - incident.resolution = Resolution.completed - incident.last_modified = ts - event.incident_id = inc_id - incident.event_ids.append(event.id) - # Status.unknown: record the event but never open/close incidents — - # a monitoring-backend failure is not a confirmed resource outage. - - self._events.append(event) - - def _make_event(self, resource_id: str, status: Status, ts: datetime.datetime, baseline: bool) -> Event: - rdef = self._defs[resource_id] - verb = "initial status" if baseline else "status changed to" - return Event( - id=str(uuid.uuid4()), - name=f"{rdef.name}: {status.value}", - description=f"{rdef.name} {verb} {status.value}.", - last_modified=ts, - occurred_at=ts, - status=status, - resource_id=resource_id, - ) - - def _make_incident(self, resource_id: str, status: Status, ts: datetime.datetime) -> Incident: - rdef = self._defs[resource_id] - return Incident( - id=str(uuid.uuid4()), - name=f"{rdef.name} {status.value}", - description=f"Automatically opened: {rdef.name} observed {status.value}.", - last_modified=ts, - status=status, - start=ts, - type=IncidentType.unplanned, - resolution=Resolution.unresolved, - resource_ids=[resource_id], - event_ids=[], - ) - - # -- readers ----------------------------------------------------------- - - def resources(self) -> list[Resource]: - out: list[Resource] = [] - for d in self._defs.values(): - out.append( - Resource( - id=d.id, - name=d.name, - description=d.description, - last_modified=self._last_modified.get(d.id, self._created_at), - site_id=self.site_id, - group=d.group, - resource_type=d.resource_type, - current_status=self._status.get(d.id, Status.unknown), - capability_ids=list(d.capability_ids), - ) - ) - return out - - def events(self) -> list[Event]: - return sorted(self._events, key=lambda e: (e.occurred_at, e.id)) - - def incidents(self) -> list[Incident]: - return sorted(self._incidents.values(), key=lambda i: (i.start, i.id)) - - -# --------------------------------------------------------------------------- -# Background poller -# --------------------------------------------------------------------------- - - -class StatusPoller: - """Periodically runs health checks and feeds results into the store.""" - - def __init__(self, store: StatusStore, settings: StatusSettings, resource_defs: list[ResourceDef]): - self.store = store - self.settings = settings - self.resource_defs = resource_defs - self._client: httpx.AsyncClient | None = None - self._checker: HealthChecker | None = None - self._task: asyncio.Task | None = None - - async def start(self) -> None: - """Create the HTTP client (inside the running loop), run one initial poll, - then launch the periodic background loop.""" - self._client = httpx.AsyncClient(verify=self.settings.tls_verify) - self._checker = HealthChecker(self.settings, self._client) - await self._poll_once() # bounded by per-query timeouts; never raises - self._task = asyncio.create_task(self._run(), name="s3df-status-poller") - logger.info( - "S3DF status poller started (%d resources, interval=%ss)", - len(self.resource_defs), - self.settings.poll_interval, - ) - - async def _run(self) -> None: - try: - while True: - await asyncio.sleep(self.settings.poll_interval) - await self._poll_once() - except asyncio.CancelledError: - raise - except Exception: # noqa: BLE001 - keep the server alive if the loop dies - logger.exception("S3DF status poller loop crashed") - - async def _poll_once(self) -> None: - assert self._checker is not None - defs = self.resource_defs - results = await asyncio.gather( - *(self._checker.check(d.health_check) for d in defs), - return_exceptions=True, - ) - for d, res in zip(defs, results): - if isinstance(res, Exception): - logger.warning("Health check raised for %s: %s", d.id, res) - continue - self.store.record(d.id, res) +from .status.config import REGISTRY, MonitoredResource, StatusSettings +from .status.poller import StatusPoller +from .status.store import StatusStore - async def aclose(self) -> None: - """Cancel the loop and close the HTTP client. For tests/lifespan wiring.""" - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - self._task = None - if self._client is not None: - await self._client.aclose() - self._client = None - - -# --------------------------------------------------------------------------- -# The adapter -# --------------------------------------------------------------------------- +logger = logging.getLogger(__name__) def _paginate(items: list, offset: int | None, limit: int | None) -> list: @@ -504,11 +60,18 @@ class S3DFStatusAdapter(status_adapter.FacilityAdapter): The status router is unauthenticated, so these methods take no user. """ - def __init__(self, settings: StatusSettings | None = None, resource_defs: list[ResourceDef] | None = None): + def __init__( + self, + settings: StatusSettings | None = None, + monitored: list[MonitoredResource] | None = None, + ): self._settings = settings or StatusSettings() - self._resource_defs = resource_defs if resource_defs is not None else RESOURCE_REGISTRY - self._store = StatusStore(site_id=self._settings.site_id, resource_defs=self._resource_defs) - self._poller = StatusPoller(self._store, self._settings, self._resource_defs) + self._monitored = monitored if monitored is not None else REGISTRY + self._store = StatusStore( + site_id=self._settings.site_id, + resources=[m.resource for m in self._monitored], + ) + self._poller = StatusPoller(self._store, self._settings, self._monitored) self._started = False self._start_lock = asyncio.Lock() diff --git a/design-docs/s3df-status-adapter-alternatives.md b/design-docs/s3df-status-adapter-alternatives.md new file mode 100644 index 00000000..12a1f371 --- /dev/null +++ b/design-docs/s3df-status-adapter-alternatives.md @@ -0,0 +1,250 @@ +# S3DF Status Adapter — Alternative Approaches Evaluation + +> Companion to `s3df-status-adapter.md`. That document describes the **as-built** +> adapter (in-process background poller + in-memory store). This document +> evaluates **alternative implementations** against the goals that matter for this +> service: low **API latency**, **simplicity**, **maintainability**, +> **extensibility**, and **scalability** (concurrent requests serviceable). + +--- + +## 1. Grounding: how this service is actually served + +``` + gunicorn.config.py: + workers = 8 + worker_class = "uvicorn.workers.UvicornWorker" +``` + +Production runs **8 independent worker processes** (the Dockerfile's `fastapi run` +is single-process, but the gunicorn config is the deployment target). This single +fact dominates the design trade-offs: + +``` + ┌──────────── load balancer ────────────┐ + │ │ │ ... │ │ + worker1 worker2 worker3 ... worker8 + │ │ │ │ + each one: its OWN in-memory store, its OWN poller, + its OWN incident/event UUIDs +``` + +With the current poller+store design, at 8 workers you get: + +- **8× backend polling load** (every worker scrapes Prometheus/InfluxDB on its own + timer), and +- **8 divergent histories** — a client hitting `/status/incidents` gets a different + answer depending on which worker the load balancer picked, with different + incident IDs for the *same* outage. + +So the real question is **not** "poller vs on-demand" — it is **"where do current +status and history actually live?"** Everything below is organized around that. + +--- + +## 2. The options at a glance + +| # | Approach | Request-path I/O | History lives in | New deps | Multi-worker consistent | +|---|----------|------------------|------------------|----------|-------------------------| +| **A** | Background poller + in-memory store *(current/as-built)* | none (RAM read) | process memory | none | ❌ diverges ×8 | +| **B** | On-demand query + TTL cache | backend on cache-miss | nowhere (current only) | none | ⚠️ current-only | +| **C** | **Prometheus as source of truth** + thin TTL cache | 1–2 Prom queries / miss | Prometheus TSDB | none | ✅ all read same TSDB | +| **D** | Read existing `s3df-status` git logs | git pull (cached) | git CSV logs | none | ✅ shared clone | +| **E** | **Proxy Alertmanager** (incidents) + Prom instant (status) | Alertmanager / Prom | Alertmanager | none\* | ✅ | +| **F** | Single poller → shared Redis/SQLite, workers read | cache read | shared store | Redis/SQLite | ✅ | + +\* Conditional on Alertmanager being deployed — standard alongside Prometheus, but +needs confirmation at S3DF. + +--- + +## 3. Per-approach review + +### A — Background poller + in-memory store (as-built) + +The data plane is built in-process: registry + async health checker + in-memory +store + background poller + a transition state machine + lazy lifecycle. + +- **Latency:** best possible — request handlers do pure in-RAM dict/list filtering, + no I/O on the hot path. +- **Concurrency:** highest (see §4). +- **Simplicity:** lowest — most moving parts of any option. +- **Maintainability:** the transition/incident state machine and the lazy + start/shutdown lifecycle are the parts most likely to harbor bugs. +- **Extensibility:** adding a resource = one registry entry (good); changing + *semantics* (flap suppression, planned maintenance) means editing the state + machine. +- **Correctness:** weakest at 8 workers — volatile (lost on restart), 8× scrape + load, and divergent per-worker histories/IDs. + +> The complexity in A exists **only** to maintain history inside the process. If +> history can live somewhere that already persists it, most of this code deletes. + +### B — On-demand query + TTL cache + +Drop the store and poller; on each request (cache-miss) query the backend for the +current value, cache the result for N seconds. + +- **Latency:** good when warm; a backend round-trip on cold/expired cache. +- **Simplicity:** much less code than A. +- **Fatal limitation:** **`/events` and `/incidents` become meaningless.** With no + polling, an outage that starts and ends between requests is never observed — + there is no continuous sampling to detect transitions. +- **Verdict:** only viable if the API is narrowed to **current status only**. + +### C — Prometheus as the source of truth (recommended default) + +Prometheus already stores the time-series **and** its history. Derive everything +from it instead of re-implementing a store: + +``` + GET /status/resources ──► Prometheus INSTANT query (/api/v1/query) + one query, labels return ALL resources at once + │ + ▼ current_status per resource + + GET /status/events ──► Prometheus RANGE query (/api/v1/query_range) + fold the series → detect transitions on the fly + │ + ▼ Event list (derived) + + GET /status/incidents ──► same RANGE data → collapse sustained "down" + windows into Incidents (start = first down, + end = recovery) + │ + ▼ Incident list (derived) + + ── all wrapped in a 15–30s TTL response cache ── +``` + +- **Latency:** near-RAM in the common case (served from the TTL cache); a small, + bounded number of Prometheus queries on a miss. +- **Simplicity:** **large reduction** — no store, no poller, no transition + *machine to maintain as state*; just a query builder + a pure transition fold + + a cache. The transition logic becomes a **pure function over data**, which is far + easier to test than a long-lived mutating state machine. +- **Maintainability/extensibility:** new resource = new query/labels; semantics + (e.g. "down for ≥ 2 samples") are expressed as query/fold parameters. +- **Scalability/consistency:** all 8 workers read the **same** TSDB → identical + answers, stable IDs (derive incident IDs deterministically from + resource+window). Survives restarts for free. Backend load is bounded by the + cache, not by request rate. +- **Trade-off:** event/incident fidelity is bounded by Prometheus **scrape + resolution and retention**; incident IDs must be derived deterministically (not + random) to stay stable across workers and refreshes. + +### D — Reuse the existing `s3df-status` git logs + +`status-pusher` already writes append-only CSV health logs to +`slaclab/s3df-status` (consumed by the Fettle dashboard). The adapter could clone/ +pull that repo and parse the logs. + +- **Pros:** zero new backend access; reuses a pipeline that already runs; + worker-consistent via a shared clone; history already persisted in git. +- **Cons:** git-pull latency on refresh; a thin point-in-time `ts,status,value` + data model with **no native incident semantics** (would still need a transition + fold like C, but over coarser data); couples IRI to the dashboard's repo layout. +- **Verdict:** lower-fidelity than C for the same amount of fold logic; best only + if direct Prometheus/InfluxDB access from the API is not permitted. + +### E — Proxy Alertmanager for incidents (best incident fidelity) + +If S3DF runs **Alertmanager** (the standard companion to Prometheus), it already +models exactly what IRI's status API wants: + +``` + Alertmanager alert (firing → resolved) ≈ IRI Event / Incident + Alertmanager grouping / dedup ≈ one Incident, many Events + Alertmanager SILENCE (maintenance window) ≈ IRI PLANNED Incident +``` + +- **Pros:** removes nearly all custom incident logic; gives **planned incidents** + (via silences) that A/B/C cannot produce; dedup/grouping handled upstream; + consistent across workers. +- **Cons:** depends on Alertmanager being deployed and on alert rules existing for + S3DF resources; current status may still come from a Prometheus instant query. +- **Verdict:** **highest-leverage path for `/events` + `/incidents`** if + Alertmanager is available — pair with a Prometheus instant query for + `current_status`. Confirm availability first. + +### F — Single poller → shared store, workers read + +Keep A's model but **decouple polling from the 8 workers**: one writer (a +dedicated poller task/process or k8s sidecar/CronJob) populates a shared +Redis/SQLite; all workers only read it. + +- **Pros:** keeps A's fast, RAM-like read latency; fixes divergence (1 writer, N + readers); 1× backend load instead of 8×; survives restarts (if persisted). +- **Cons:** adds an external store dependency and a deployment component. +- **Verdict:** the right evolution **if** the poller model must be retained. + +--- + +## 4. Scalability — "how many concurrent requests can we serve?" + +The deciding factor is whether the **request path performs remote I/O**: + +``` + ── RAM / cache hit (A, C-warm, D-warm, F) ───────────────────────── + handler ≈ microseconds of dict/list filtering + • ceiling = CPU × 8 workers → thousands of req/s on the event loop + • BACKEND load is CONSTANT, independent of request rate + + ── Remote on the request path (B-cold, C-cold, E) ────────────────── + per-request latency = backend round-trip (~10–50 ms Prom instant) + • concurrency bounded by httpx pool size + backend capacity + • BACKEND load SCALES with request rate (burst risk / thundering herd) +``` + +Implications: + +- **A** maximizes concurrency, but pays with **8× backend load** and inconsistent + data. +- **C with a short TTL cache** achieves **the same warm-path concurrency as A** + (reads served from cache), while collapsing backend load to a steady trickle and + staying consistent. **The cache is what lets C scale like A without A's state + machine.** +- **B-cold / C-cold / E** are bounded by the backend; protect them with the TTL + cache + single-flight (coalesce concurrent misses into one backend call) to avoid + a thundering herd when the cache expires. + +Rule of thumb: keep the **hot path in-memory/cache**, keep **backend load +decoupled from request rate**, and the service comfortably saturates the 8 workers' +CPU before any backend becomes the bottleneck. + +--- + +## 5. Recommendation (tiered) + +1. **Immediate, lowest-effort win (keep current code):** decouple polling from the + workers — run a **single** poller (option **F-lite**) or set this route's + workers to 1 — to stop the 8× scrape load and per-worker divergence **today**. + +2. **Best balance of simplicity + latency + consistency:** **migrate to C — + Prometheus as the source of truth + a 15–30s TTL cache (with single-flight).** + Deletes the store, poller, and stateful transition machine; fixes restart-loss + and worker divergence; reuses infrastructure already operated; holds latency + near-RAM via the cache. + +3. **For real incidents + planned maintenance:** **investigate E (Alertmanager).** + If deployed, proxy it for `/events` + `/incidents` (silences → planned + incidents) and keep a Prometheus instant query for `current_status`. + +### Net + +The in-memory machinery in the as-built adapter is the thing to **remove**. +Prometheus — and likely Alertmanager — already provide persistence, history, and +cross-worker consistency. Pushing state into them **simplifies the code** *and* +**improves scalability**, while a TTL cache preserves the low API latency that +the in-memory design was chosen for in the first place. + +--- + +## 6. Decision checklist (to unblock a direction) + +- [ ] Is direct **Prometheus** query access allowed from the IRI API pods? → enables **C**. +- [ ] Is **Alertmanager** deployed, with alert rules covering S3DF resources? → enables **E**. +- [ ] Must `/events` and `/incidents` be meaningful (vs. current-status-only)? → rules out **B**. +- [ ] Is adding **Redis/SQLite** acceptable operationally? → enables **F**. +- [ ] Required event/incident **fidelity** vs. Prometheus scrape interval & retention? +- [ ] Target **request rate / concurrency** and acceptable **p99 latency** under cache-miss? diff --git a/design-docs/s3df-status-adapter.md b/design-docs/s3df-status-adapter.md index 90a71d41..427b4449 100644 --- a/design-docs/s3df-status-adapter.md +++ b/design-docs/s3df-status-adapter.md @@ -327,3 +327,260 @@ app/s3df/ - `prometheus-api-client` (already used by status-pusher) - `httpx` (async HTTP for InfluxDB queries — preferred over sync `requests` in async app) - No new external services required; uses existing Prometheus + InfluxDB infrastructure + +--- + +> **As-built addendum.** Sections 11–12 below describe the logic actually +> implemented in `app/s3df/status_adapter.py`. The adapter ships as a **single +> file** (registry + health checker + store + poller + adapter class), not the +> multi-module layout sketched in sections 5/8. Prometheus is queried directly +> over its HTTP API via `httpx` (the `prometheus-api-client` dependency was not +> needed). + +## 11. Status Computation (As Built) + +### 11.1 End-to-end pipeline + +A resource's `current_status` is derived once per poll cycle by turning a single +scalar metric into one of four `Status` values. The same pipeline runs for every +resource in `RESOURCE_REGISTRY`. + +``` + ResourceDef.health_check + ┌───────────────────────────────────────────────────────────────────────┐ + │ backend (prometheus|influxdb) + query + up_when [+ degraded_when] │ + └───────────────────────────────────────────────────────────────────────┘ + │ + ▼ HealthChecker.check() + ┌──────────────────┐ query backend over httpx (timeout-bounded) + │ Prometheus │──► GET /api/v1/query?query=... + │ or InfluxDB │──► GET /query?q=...&db=... + └──────────────────┘ + │ + ▼ parse a single scalar + value: float | None (None = empty result / parse miss) + │ + ▼ evaluate(check, value) + ┌───────────────────────────┐ + │ Status: │ + │ up | degraded | down | │ + │ unknown │ + └───────────────────────────┘ + │ + ▼ StatusStore.record(resource_id, HealthResult) + update current_status, emit Event on change, open/close Incident +``` + +### 11.2 Value → Status rule + +`evaluate()` applies the resource's conditions in a fixed precedence. A +`Condition` is just a comparator (`eq/ne/lt/lte/gt/gte`) applied to the observed +value against a threshold — the same success-criterion idea `status-pusher` uses. + +``` + value is None ───────────────────────────────► UNKNOWN + │ (query failed, errored, empty result, + │ or backend mapped to no scalar) + ▼ + up_when.met(value)? ── yes ──────────────────► UP + │ no + ▼ + degraded_when set AND degraded_when.met(value)? ─ yes ──► DEGRADED + │ no + ▼ + DOWN +``` + +Notes: +- **`unknown` ≠ `down`.** Any exception during the query (timeout, TLS, HTTP + error, malformed JSON) is caught and mapped to `unknown`, *not* `down`. A + monitoring-plane failure is not treated as a confirmed resource outage. +- **`degraded` is optional.** Resources that only define `up_when` collapse to a + binary up/down model (matching `status-pusher`). `degraded_when` is the hook + for a future third threshold. +- The registry currently uses `up_when = (value == 1.0)` for all resources + (nmap port state for SSH; monit `status_code` for slurmctld/slurmdbd). + +### 11.3 Transition detection & events + +The store is **change-driven**: `record()` compares the new status to the +resource's previous status and does nothing on a steady state. Events are emitted +only on a real change, which keeps the event log meaningful instead of one row +per poll. + +``` + poll cycle N: prev = store.current_status[r] + new = result.status + + prev == new ────────────────► no-op (steady state) + + prev != new ────────────────► emit Event(occurred_at = poll time, + status = new) + + drive incident state machine (11.4) + + Special case: prev is None (first ever observation) + → "baseline" Event describing the initial status + → if initial status is down/degraded, an incident is ALSO opened + (its start time is the first-observed time, not the true outage start) +``` + +Example for one resource over six polls: + +``` + poll: 1 2 3 4 5 6 + status: up up down down up unknown + event: base — E1 — E2 E3 + ▲ ▲ ▲ + │ │ └ up→unknown (incident stays open? no — + │ │ it was already closed at poll 5) + │ └ down→up (closes incident, resolution=completed) + └ up→down (opens unplanned incident) +``` + +### 11.4 Incident lifecycle (state machine) + +At most **one open incident per resource**. Incidents are auto-created as +`unplanned`. The driving signal is the resource's status transition: + +``` + ┌───────────────────────────────────────────┐ + │ │ + up / (start) │ │ + ───────────────► (NO OPEN INCIDENT) │ + │ │ │ + down|degraded │ │ down|degraded │ up + │ ▼ │ + (OPEN: unplanned, resolution=unresolved) │ + │ │ ▲ │ + down⇄degraded │ │ down|degraded (escalate: │ + (update incident │ │ update incident.status only) │ + status + mtime) │ └─────────────────────────────── │ + │ │ │ + │ │ up ─────────────────────────────────┘ + │ ▼ close: end=now, + │ (CLOSED: resolution=completed, status=up) + │ + unknown ───────┘ (incident is LEFT OPEN; only an event is recorded — + a backend failure must not auto-resolve an outage) +``` + +Concretely, in `record()`: +- **→ down/degraded:** open a new incident if none is open for the resource; + otherwise update the existing incident's `status`. Link the new event to the + incident (`event.incident_id`, `incident.event_ids`). +- **→ up:** pop and close the open incident (`end`, `resolution=completed`, + `status=up`), linking the closing event. +- **→ unknown:** record the event only; never opens or closes an incident. + +### 11.5 Polling cadence & lazy start + +IRI constructs adapters **synchronously at import time**, before the asyncio loop +exists, so the poller cannot start in `__init__`. Instead it starts lazily on the +first request, guarded by a double-checked `asyncio.Lock`. + +``` + import time first request (loop running) steady state + ───────────────► ─────────────────────────► ───────────► + + __init__: _ensure_started(): background task: + build store, async with start_lock: loop forever: + poller, lock if not started: sleep(interval) + (NO network, • create httpx.AsyncClient poll_once() + NO task) • await poll_once() ◄─ initial └ gather all + (populates store so the checks, record + first response isn't empty) + • create_task(_run()) + • started = True + ── request proceeds, reads store ── +``` + +- The **initial poll is awaited** under the lock, so the very first + `/status/...` response reflects real data rather than all-`unknown`. Its cost is + bounded by per-query `S3DF_STATUS_HTTP_TIMEOUT`, and `check()` never raises + (failures become `unknown`). +- All resource checks within a cycle run concurrently via `asyncio.gather`. +- `aclose()` cancels the loop and closes the HTTP client (for tests / future + FastAPI-lifespan wiring). + +## 12. Limitations & Known Trade-offs + +These are intentional simplifications for a first reviewable implementation. Each +has a clear upgrade path. + +### 12.1 Volatile, per-process state + +State (current status, events, incidents) lives in plain Python structures. It is +**lost on restart** and **diverges across uvicorn workers** — each worker polls +independently and mints its own incident/event UUIDs, so a client may see +different histories depending on which worker answers. + +``` + ┌── worker A ──┐ ┌── worker B ──┐ + │ store_A │ │ store_B │ + │ inc id=abc │ ≠ │ inc id=xyz │ ← same outage, different ids + └──────────────┘ └──────────────┘ + GET /status/incidents → load-balanced → answer depends on worker +``` + +Upgrade path: shared store (SQLite/Redis/Postgres) or a single dedicated poller +process writing a store the API workers only read. + +### 12.2 Single sample, no flap suppression + +Status is decided from **one** sample per cycle with no debounce/hysteresis. A +single transient bad scrape flips the resource and generates an event/incident; +the next good poll closes it. A flapping metric produces incident churn. + +``` + value: 1 1 0 1 0 1 (0 = "bad" scrape) + status: up up dn up dn up + events: — E E E E ← four events, two short incidents +``` + +Upgrade path: require N consecutive failures before opening (and N successes +before closing), or evaluate over a rolling window. + +### 12.3 `unknown` and unknown outage-start + +- A backend/query failure yields `unknown`, which deliberately does **not** close + an open incident — but a long backend outage will leave a resource pinned at + `unknown` with no incident of its own. +- When a resource is **already down at first observation**, the incident `start` + is set to *first-observed time*, which is not the true outage start (the + adapter has no history before it booted). + +### 12.4 Simplistic query → scalar reduction + +Each backend response is reduced to a single number: Prometheus +`data.result[0].value[1]`, InfluxDB `series[0].values[-1][1]`. Queries that return +multiple series / grouped results (e.g. `GROUP BY service`) only have their +**first** series considered. Health checks must be written to return a single +series. There is no "any/all/aggregate across series" policy yet. + +### 12.5 Scope gaps + +- **No planned incidents.** All incidents are `unplanned`; there is no ingestion + of maintenance windows (`planned` / `reservation`). Planned-maintenance support + needs a schedule source (config or API). +- **`site_id` is not a guaranteed cross-reference.** Resources carry + `S3DF_SITE_ID` (default `"s3df"`), but the `/facility` adapter currently mints a + random site UUID per process, so `Resource.site_uri` may not resolve to a real + site until a stable site id is wired through. +- **Resource set is minimal.** Only SSH gateway + slurmctld + slurmdbd are + registered today; the full S3DF resource catalogue (storage, network, web + services, …) still needs to be enumerated. + +### 12.6 Operational + +- **Unbounded event growth.** The event log is append-only and never trimmed; + a long-lived process accumulates events in memory. Needs retention/eviction + (or persistence with TTL). +- **First-request latency.** The first caller after startup pays the awaited + initial poll (up to `S3DF_STATUS_HTTP_TIMEOUT` per backend round). +- **TLS verification defaults off.** `S3DF_STATUS_TLS_VERIFY` defaults to `false` + (SLAC internal CA convenience); set it to `true` or a CA-bundle path in + production. +- **No automatic shutdown wiring.** `aclose()` exists but the framework does not + call it; wiring it into FastAPI's lifespan is a follow-up so the poller task and + HTTP client are closed cleanly on shutdown. + From b17c6b8f3d710cbb4fc2198ba017a933103b1985 Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Mon, 1 Jun 2026 16:59:44 -0700 Subject: [PATCH 4/6] updated resources and status polling mechanism --- app/routers/status/status.py | 28 +- app/s3df/status/__init__.py | 7 +- app/s3df/status/config.py | 303 ++++++-- app/s3df/status/health_checker.py | 105 ++- app/s3df/status/poller.py | 4 +- app/s3df/status_adapter.py | 23 +- app/s3df/tests/test_status_adapter.py | 307 ++++++++ .../s3df-status-adapter-alternatives.md | 14 +- design-docs/s3df-status-adapter.md | 728 ++++-------------- local-template.env | 3 + 10 files changed, 813 insertions(+), 709 deletions(-) create mode 100644 app/s3df/tests/test_status_adapter.py diff --git a/app/routers/status/status.py b/app/routers/status/status.py index ce5958e4..0ceceba2 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -36,10 +36,34 @@ async def get_resources( resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), capability: List[AllocationUnit] = Query(default=None, min_length=1), - _forbid=Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), + site_id: str | None = Query(default=None, min_length=1), + _forbid=Depends( + forbidExtraQueryParams( + "name", + "description", + "group", + "offset", + "limit", + "modified_since", + "resource_type", + "current_status", + "capability", + "site_id", + multiParams={"capability"}, + ) + ), ) -> list[models.Resource]: return await router.adapter.get_resources( - offset=offset, limit=limit, name=name, description=description, group=group, modified_since=modified_since, resource_type=resource_type, current_status=current_status, capability=capability + offset=offset, + limit=limit, + name=name, + description=description, + group=group, + modified_since=modified_since, + resource_type=resource_type, + current_status=current_status, + capability=capability, + site_id=site_id, ) diff --git a/app/s3df/status/__init__.py b/app/s3df/status/__init__.py index 8ffc082a..9687cc89 100644 --- a/app/s3df/status/__init__.py +++ b/app/s3df/status/__init__.py @@ -4,7 +4,7 @@ Splits the adapter into focused modules: * ``config`` — health-check model, resource registry, settings - * ``health_checker`` — Prometheus/InfluxDB query + status evaluation + * ``health_checker`` — Prometheus, InfluxDB, HTTP query + evaluation * ``store`` — in-memory current-status / event / incident store * ``poller`` — background polling loop @@ -18,8 +18,9 @@ HealthCheck, MonitoredResource, StatusSettings, + build_registry, ) -from .health_checker import HealthChecker, HealthResult, evaluate +from .health_checker import HealthChecker, HealthResult, aggregate_results, evaluate from .poller import StatusPoller from .store import StatusStore @@ -30,8 +31,10 @@ "HealthCheck", "MonitoredResource", "StatusSettings", + "build_registry", "HealthChecker", "HealthResult", + "aggregate_results", "evaluate", "StatusPoller", "StatusStore", diff --git a/app/s3df/status/config.py b/app/s3df/status/config.py index bf29c94a..c4658e2f 100644 --- a/app/s3df/status/config.py +++ b/app/s3df/status/config.py @@ -1,40 +1,29 @@ """ Configuration for the S3DF status adapter. -Holds the static, request-independent pieces of the adapter: - - * The health-check model (``Backend``, ``Condition``, ``HealthCheck``) — a - declarative success-criterion mirroring the model ``status-pusher`` uses. - * ``MonitoredResource`` — pairs an IRI ``Resource`` template with the health - check that drives it (see the duplication note below). - * ``REGISTRY`` — the set of S3DF resources surfaced by ``/status``. - * ``StatusSettings`` — environment-driven configuration. - -Avoiding Resource/ResourceDef duplication ------------------------------------------- -A resource's stable identity/metadata (id, name, description, group, -resource_type, capability_ids) is declared exactly once, as an IRI -``Resource`` template inside ``MonitoredResource``. The runtime-owned fields -(``current_status``, ``last_modified``, ``site_id``) are placeholders here and -are overlaid by the store when it projects a live view, so the fields are never -re-listed in a parallel definition. +IRI owns the S3DF status runtime: it periodically runs configured health checks, +evaluates status-pusher-style conditions, and caches the latest result for each +resource. The external S3DF status repositories are reference material only; this +module does not fetch dashboard log files. Required/optional env vars: - S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) - S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) - S3DF_INFLUXDB_DB InfluxDB database (default telegraf) - S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) - S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) - S3DF_SITE_ID Site id for resources (default s3df) - S3DF_STATUS_TLS_VERIFY true | false | (default false) + S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) + S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) + S3DF_INFLUXDB_DB InfluxDB database (default telegraf) + S3DF_STATUS_CHECKS_JSON JSON mapping resource ids to additional health checks + S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) + S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) + S3DF_SITE_ID Site id for resources (default s3df) + S3DF_STATUS_TLS_VERIFY true | false | (default false) """ import datetime +import json import operator import os -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Callable +from typing import Any, Callable from app.routers.status.models import Resource, ResourceType, Status @@ -43,16 +32,15 @@ def utc_now() -> datetime.datetime: return datetime.datetime.now(datetime.timezone.utc) -# Placeholder timestamp baked into static Resource templates. The store is the -# authority for ``last_modified`` and overwrites this when projecting a view. _EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) class Backend(str, Enum): - """Metrics backend a health check queries.""" + """Backend/source a health check queries.""" prometheus = "prometheus" influxdb = "influxdb" + http = "http" _COMPARATORS: dict[str, Callable[[float, float], bool]] = { @@ -67,42 +55,41 @@ class Backend(str, Enum): @dataclass(frozen=True) class Condition: - """A comparison of an observed metric value against a threshold.""" + """A comparison of an observed value against a threshold.""" comparator: str value: float - def met(self, observed: float) -> bool: - op = _COMPARATORS.get(self.comparator) - if op is None: + def __post_init__(self) -> None: + if self.comparator not in _COMPARATORS: raise ValueError(f"Unknown comparator: {self.comparator}") - return bool(op(observed, self.value)) + + def met(self, observed: float) -> bool: + return bool(_COMPARATORS[self.comparator](observed, self.value)) @dataclass(frozen=True) class HealthCheck: - """How to determine a resource's status from a metrics backend.""" + """How to determine a resource's status from an IRI-owned source.""" backend: Backend - query: str - up_when: Condition - db_name: str | None = None # InfluxDB only + name: str | None = None + query: str | None = None + up_when: Condition = field(default_factory=lambda: Condition("eq", 1.0)) + db_name: str | None = None degraded_when: Condition | None = None + url: str | None = None + method: str = "GET" + headers: dict[str, str] = field(default_factory=dict) + follow_redirects: bool = True @dataclass(frozen=True) class MonitoredResource: - """Pairs a static ``Resource`` template with the health check driving it. - - The ``Resource`` carries the resource's stable identity/metadata exactly - once. Its dynamic fields (``current_status``, ``last_modified``, - ``site_id``) are placeholders that the store overlays when it builds the - live view, which is what lets us avoid a separate ``ResourceDef`` that would - otherwise re-list the same fields. - """ + """Pairs a static Resource template with the health checks driving it.""" resource: Resource - health_check: HealthCheck + health_checks: tuple[HealthCheck, ...] = () def _template( @@ -114,14 +101,13 @@ def _template( resource_type: ResourceType, capability_ids: tuple[str, ...] = (), ) -> Resource: - """Build a static Resource template. Dynamic fields are placeholders the - store overwrites (``current_status``, ``last_modified``, ``site_id``).""" + """Build a static Resource template. The store overwrites dynamic fields.""" return Resource( id=id, name=name, description=description, last_modified=_EPOCH, - site_id="", # overlaid by the store from settings.site_id + site_id="", group=group, resource_type=resource_type, current_status=Status.unknown, @@ -129,33 +115,101 @@ def _template( ) -# The set of S3DF resources surfaced by /status. Queries mirror those exercised -# by status-pusher's live tests (see status-pusher/Makefile). Extend as needed. -REGISTRY: list[MonitoredResource] = [ - MonitoredResource( - resource=_template( - id="s3df-ssh-gateway", - name="SSH Login Gateway", - description="S3DF interactive SSH login gateway reachability.", - group="access", - resource_type=ResourceType.service, - ), - health_check=HealthCheck( +RESOURCE_TEMPLATES: tuple[Resource, ...] = ( + _template( + id="s3df-ssh-bastions", + name="SSH Bastions", + description="S3DF SSH bastion hosts for command-line access.", + group="access", + resource_type=ResourceType.service, + ), + _template( + id="s3df-interactive-nodes", + name="Interactive Nodes", + description="S3DF interactive login and analysis nodes.", + group="compute", + resource_type=ResourceType.compute, + ), + _template( + id="s3df-docs", + name="S3DF Docs", + description="S3DF user documentation site.", + group="documentation", + resource_type=ResourceType.website, + ), + _template( + id="s3df-batch-servers", + name="Batch Servers", + description="S3DF batch submission and scheduling servers.", + group="compute", + resource_type=ResourceType.compute, + ), + _template( + id="s3df-slurm", + name="Slurm", + description="S3DF Slurm workload management service.", + group="compute", + resource_type=ResourceType.service, + ), + _template( + id="s3df-monitoring", + name="Monitoring", + description="S3DF monitoring and observability services.", + group="operations", + resource_type=ResourceType.service, + ), + _template( + id="s3df-coact", + name="Coact", + description="S3DF Coact allocation and account service.", + group="accounts", + resource_type=ResourceType.service, + ), + _template( + id="s3df-ondemand", + name="OnDemand", + description="S3DF Open OnDemand web service.", + group="access", + resource_type=ResourceType.website, + ), + _template( + id="s3df-kubernetes", + name="Kubernetes", + description="S3DF Kubernetes platform.", + group="platform", + resource_type=ResourceType.system, + ), + _template( + id="s3df-storage", + name="Storage", + description="S3DF storage services.", + group="storage", + resource_type=ResourceType.storage, + ), + _template( + id="s3df-dtns", + name="DTNs", + description="S3DF data transfer nodes.", + group="data-transfer", + resource_type=ResourceType.network, + ), +) + +RESOURCE_IDS = {resource.id for resource in RESOURCE_TEMPLATES} + +BUILTIN_CHECKS: dict[str, tuple[HealthCheck, ...]] = { + "s3df-ssh-bastions": ( + HealthCheck( backend=Backend.prometheus, + name="ssh-bastion-port-state", query="avg( avg_over_time(nmap_port_state{service=`ssh`,group=`s3df`}[5m]) )", up_when=Condition("eq", 1.0), ), ), - MonitoredResource( - resource=_template( - id="s3df-slurmctld", - name="Slurm Controller (slurmctld)", - description="Slurm workload manager controller daemon health.", - group="compute", - resource_type=ResourceType.compute, - ), - health_check=HealthCheck( + "s3df-slurm": ( + HealthCheck( backend=Backend.influxdb, + name="slurmctld-process", db_name="telegraf", query=( 'SELECT mean("status_code") FROM "monit_process" ' @@ -163,17 +217,9 @@ def _template( ), up_when=Condition("eq", 1.0), ), - ), - MonitoredResource( - resource=_template( - id="s3df-slurmdbd", - name="Slurm DB Daemon (slurmdbd)", - description="Slurm accounting database daemon health.", - group="compute", - resource_type=ResourceType.compute, - ), - health_check=HealthCheck( + HealthCheck( backend=Backend.influxdb, + name="slurmdbd-process", db_name="telegraf", query=( 'SELECT mean("status_code") FROM "monit_process" ' @@ -182,7 +228,23 @@ def _template( up_when=Condition("eq", 1.0), ), ), -] +} + + +def build_registry(settings: "StatusSettings | None" = None) -> list[MonitoredResource]: + """Build monitored resources with built-in plus configured checks.""" + configured = settings.resource_checks if settings is not None else {} + return [ + MonitoredResource( + resource=resource, + health_checks=BUILTIN_CHECKS.get(resource.id, ()) + configured.get(resource.id, ()), + ) + for resource in RESOURCE_TEMPLATES + ] + + +# Static default registry for tests/importers that do not need env-driven checks. +REGISTRY: list[MonitoredResource] = build_registry() class StatusSettings: @@ -192,6 +254,7 @@ def __init__(self) -> None: self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") + self.resource_checks = self._parse_resource_checks(os.getenv("S3DF_STATUS_CHECKS_JSON", "{}")) self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) # NOTE: the /facility adapter currently mints a random site uuid per @@ -207,5 +270,79 @@ def _parse_verify(raw: str) -> bool | str: return True if low in ("false", "0", "no", "off", ""): return False - # Anything else is treated as a path to a CA bundle. return raw + + @classmethod + def _parse_resource_checks(cls, raw: str) -> dict[str, tuple[HealthCheck, ...]]: + raw = raw.strip() + if not raw: + return {} + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("S3DF_STATUS_CHECKS_JSON must be valid JSON") from exc + if not isinstance(data, dict): + raise ValueError("S3DF_STATUS_CHECKS_JSON must be an object keyed by resource id") + + parsed: dict[str, tuple[HealthCheck, ...]] = {} + for resource_id, checks in data.items(): + if resource_id not in RESOURCE_IDS: + raise ValueError(f"Unknown S3DF status resource id in S3DF_STATUS_CHECKS_JSON: {resource_id}") + if not isinstance(checks, list): + raise ValueError(f"Checks for {resource_id} must be a list") + parsed[resource_id] = tuple(cls._parse_check(resource_id, idx, check) for idx, check in enumerate(checks)) + return parsed + + @classmethod + def _parse_check(cls, resource_id: str, idx: int, raw: Any) -> HealthCheck: + if not isinstance(raw, dict): + raise ValueError(f"Check {idx} for {resource_id} must be an object") + try: + backend = Backend(raw["backend"]) + except KeyError as exc: + raise ValueError(f"Check {idx} for {resource_id} is missing backend") from exc + except ValueError as exc: + raise ValueError(f"Check {idx} for {resource_id} has unsupported backend: {raw.get('backend')}") from exc + + up_when = cls._parse_condition(raw.get("up_when"), default=Condition("eq", 200.0 if backend == Backend.http else 1.0)) + degraded_when = cls._parse_condition(raw.get("degraded_when"), default=None) + headers = raw.get("headers", {}) + if headers is None: + headers = {} + if not isinstance(headers, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in headers.items()): + raise ValueError(f"Check {idx} for {resource_id} has invalid headers") + + check = HealthCheck( + backend=backend, + name=raw.get("name"), + query=raw.get("query"), + up_when=up_when, + db_name=raw.get("db_name"), + degraded_when=degraded_when, + url=raw.get("url"), + method=str(raw.get("method", "GET")).upper(), + headers=headers, + follow_redirects=bool(raw.get("follow_redirects", True)), + ) + cls._validate_check(resource_id, idx, check) + return check + + @staticmethod + def _parse_condition(raw: Any, default: Condition | None) -> Condition | None: + if raw is None: + return default + if not isinstance(raw, dict): + raise ValueError("condition must be an object") + try: + comparator = raw["comparator"] + value = raw["value"] + except KeyError as exc: + raise ValueError("condition requires comparator and value") from exc + return Condition(str(comparator), float(value)) + + @staticmethod + def _validate_check(resource_id: str, idx: int, check: HealthCheck) -> None: + if check.backend in (Backend.prometheus, Backend.influxdb) and not check.query: + raise ValueError(f"Check {idx} for {resource_id} requires query") + if check.backend == Backend.http and not check.url: + raise ValueError(f"Check {idx} for {resource_id} requires url") diff --git a/app/s3df/status/health_checker.py b/app/s3df/status/health_checker.py index 7d68020f..e34e5f00 100644 --- a/app/s3df/status/health_checker.py +++ b/app/s3df/status/health_checker.py @@ -1,14 +1,16 @@ """ Health checker for the S3DF status adapter. -Runs a :class:`~app.s3df.status.config.HealthCheck` against its metrics backend -(Prometheus or InfluxDB) over ``httpx`` and reduces the response to a single -:class:`~app.routers.status.models.Status` via :func:`evaluate`. +Runs status-pusher-like checks from IRI-owned configuration against Prometheus, +InfluxDB, or HTTP endpoints, evaluates conditions, and aggregates check results +into one resource status. """ +import asyncio import datetime import logging from dataclasses import dataclass +from typing import Sequence import httpx @@ -21,7 +23,7 @@ @dataclass class HealthResult: - """Outcome of a single health check.""" + """Outcome of one resource health evaluation.""" status: Status value: float | None @@ -30,7 +32,7 @@ class HealthResult: def evaluate(check: HealthCheck, value: float | None) -> Status: - """Map an observed metric value to a Status using the check's conditions.""" + """Map an observed scalar value to a Status using the check's conditions.""" if value is None: return Status.unknown if check.up_when.met(value): @@ -40,63 +42,106 @@ def evaluate(check: HealthCheck, value: float | None) -> Status: return Status.down -class HealthChecker: - """Runs a HealthCheck against its backend and evaluates the result. +def aggregate_results(results: Sequence[HealthResult]) -> HealthResult: + """Aggregate multiple check results into one resource status. - The caller owns the ``httpx.AsyncClient`` lifecycle (created from within the - running event loop) and passes it in. + Unknown checks are ignored when at least one check produced a usable signal. + This avoids reporting an outage just because one redundant probe failed. """ + observed_at = max((result.observed_at for result in results), default=utc_now()) + if not results: + return HealthResult(Status.unknown, None, observed_at, "no health checks configured") + + known = [result for result in results if result.status != Status.unknown] + if not known: + errors = "; ".join(result.error for result in results if result.error) + return HealthResult(Status.unknown, None, observed_at, errors or "all health checks returned unknown") + + if any(result.status == Status.down for result in known): + status = Status.down + elif any(result.status == Status.degraded for result in known): + status = Status.degraded + else: + status = Status.up + + return HealthResult(status, None, observed_at) + + +class HealthChecker: + """Executes health checks over a shared httpx AsyncClient.""" - def __init__(self, settings: StatusSettings, client: httpx.AsyncClient): + def __init__(self, settings: StatusSettings, client: httpx.AsyncClient) -> None: self.settings = settings self.client = client - async def check(self, check: HealthCheck) -> HealthResult: + async def check(self, checks: Sequence[HealthCheck]) -> HealthResult: + if not checks: + return HealthResult(Status.unknown, None, utc_now(), "no health checks configured") + + results = await asyncio.gather(*(self.check_one(check) for check in checks), return_exceptions=True) + normalized: list[HealthResult] = [] + for result in results: + if isinstance(result, HealthResult): + normalized.append(result) + else: + normalized.append(HealthResult(Status.unknown, None, utc_now(), str(result))) + return aggregate_results(normalized) + + async def check_one(self, check: HealthCheck) -> HealthResult: now = utc_now() try: if check.backend == Backend.prometheus: + if check.query is None: + raise ValueError("query is required for prometheus health checks") value = await self._prometheus_query(check.query) elif check.backend == Backend.influxdb: + if check.query is None: + raise ValueError("query is required for influxdb health checks") db = check.db_name or self.settings.influxdb_db value = await self._influx_query(db, check.query) - else: # pragma: no cover - guarded by enum + elif check.backend == Backend.http: + value = await self._http_query(check) + else: # pragma: no cover - guarded by enum/config validation return HealthResult(Status.unknown, None, now, f"unknown backend {check.backend}") - except Exception as exc: # noqa: BLE001 - any failure -> unknown - logger.warning("Health check failed (%s): %s", check.backend.value, exc) + except Exception as exc: # noqa: BLE001 - health polling should surface errors as unknown status + label = f"{check.backend.value}:{check.name}" if check.name else check.backend.value + logger.warning("Health check failed (%s): %s", label, exc) return HealthResult(Status.unknown, None, now, str(exc)) return HealthResult(evaluate(check, value), value, now) async def _prometheus_query(self, query: str) -> float | None: - """Instant query via the Prometheus HTTP API; returns the scalar value.""" url = self.settings.prometheus_url.rstrip("/") + "/api/v1/query" resp = await self.client.get(url, params={"query": query}, timeout=self.settings.http_timeout) resp.raise_for_status() data = resp.json() if data.get("status") != "success": - raise RuntimeError(f"prometheus error: {data.get('error', 'unknown')}") + raise RuntimeError(f"Prometheus returned status={data.get('status')}") result = data.get("data", {}).get("result", []) if not result: return None - # Instant vector sample: value == [epoch_ts, "stringified_value"] return float(result[0]["value"][1]) - async def _influx_query(self, db_name: str, query: str) -> float | None: - """Query the InfluxDB HTTP API; returns the most recent scalar value.""" + async def _influx_query(self, db: str, query: str) -> float | None: url = self.settings.influxdb_url.rstrip("/") + "/query" - resp = await self.client.get( - url, params={"q": query, "db": db_name}, timeout=self.settings.http_timeout - ) + resp = await self.client.get(url, params={"db": db, "q": query}, timeout=self.settings.http_timeout) resp.raise_for_status() data = resp.json() results = data.get("results", []) if not results: return None - series = results[0].get("series") - if not series: - return None - values = series[0].get("values") - if not values: + series = results[0].get("series", []) + if not series or not series[0].get("values"): return None - # Each row is [time, value]; take the latest row's value column. - row = values[-1] - return float(row[1]) if row[1] is not None else None + return float(series[0]["values"][-1][1]) + + async def _http_query(self, check: HealthCheck) -> float: + if check.url is None: + raise ValueError("url is required for http health checks") + response = await self.client.request( + check.method, + check.url, + headers=check.headers, + timeout=self.settings.http_timeout, + follow_redirects=check.follow_redirects, + ) + return float(response.status_code) diff --git a/app/s3df/status/poller.py b/app/s3df/status/poller.py index 56584cf0..63df3e82 100644 --- a/app/s3df/status/poller.py +++ b/app/s3df/status/poller.py @@ -1,7 +1,7 @@ """ Background poller for the S3DF status adapter. -Periodically runs the health check for each monitored resource and feeds the +Periodically runs the health checks for each monitored resource and feeds the results into the :class:`~app.s3df.status.store.StatusStore`. The ``httpx`` client is created lazily inside the running event loop (see :meth:`start`). """ @@ -56,7 +56,7 @@ async def _poll_once(self) -> None: assert self._checker is not None monitored = self.monitored results = await asyncio.gather( - *(self._checker.check(m.health_check) for m in monitored), + *(self._checker.check(m.health_checks) for m in monitored), return_exceptions=True, ) for m, res in zip(monitored, results): diff --git a/app/s3df/status_adapter.py b/app/s3df/status_adapter.py index 650dd86a..cc5d5a18 100644 --- a/app/s3df/status_adapter.py +++ b/app/s3df/status_adapter.py @@ -1,19 +1,18 @@ """ SLAC S3DF Status Adapter for the IRI Facility API. -Serves the IRI ``/status`` router (resources, events, incidents) from live -health checks against S3DF monitoring infrastructure — Prometheus and InfluxDB -(telegraf) — the same backends the standalone ``status-pusher`` tool queries. +Serves the IRI ``/status`` router (resources, events, incidents) from +IRI-owned periodic health checks and a local in-memory status cache. Design (see design-docs/s3df-status-adapter.md): - * Resources are STATIC config (``app.s3df.status.config.REGISTRY``). Each - pairs an IRI ``Resource`` template with a health check (backend + query + - condition) — mirroring the success-criterion model used by ``status-pusher``. + * Resources are STATIC config (``app.s3df.status.config``). Each pairs an IRI + ``Resource`` template with zero or more health checks. Built-in checks are + merged with optional ``S3DF_STATUS_CHECKS_JSON`` checks at adapter startup. * A background poller (``app.s3df.status.poller``) runs every - ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries each resource's backend, maps - the result to a Status (up/down/degraded/unknown), and records it in an - in-memory store (``app.s3df.status.store``). + ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries each resource's configured + sources, maps the aggregate result to a Status (up/down/degraded/unknown), + and records it in an in-memory store (``app.s3df.status.store``). * The store detects status TRANSITIONS to emit Events and to open/close unplanned Incidents. @@ -35,7 +34,7 @@ from app.routers.status import facility_adapter as status_adapter from app.routers.status import models as status_models -from .status.config import REGISTRY, MonitoredResource, StatusSettings +from .status.config import MonitoredResource, StatusSettings, build_registry from .status.poller import StatusPoller from .status.store import StatusStore @@ -51,7 +50,7 @@ def _paginate(items: list, offset: int | None, limit: int | None) -> list: class S3DFStatusAdapter(status_adapter.FacilityAdapter): - """IRI status FacilityAdapter backed by live S3DF health checks. + """IRI status FacilityAdapter backed by internal S3DF health checks. Implements every method called by the IRI status router: get_resources, get_resource, get_events, get_event, @@ -66,7 +65,7 @@ def __init__( monitored: list[MonitoredResource] | None = None, ): self._settings = settings or StatusSettings() - self._monitored = monitored if monitored is not None else REGISTRY + self._monitored = monitored if monitored is not None else build_registry(self._settings) self._store = StatusStore( site_id=self._settings.site_id, resources=[m.resource for m in self._monitored], diff --git a/app/s3df/tests/test_status_adapter.py b/app/s3df/tests/test_status_adapter.py new file mode 100644 index 00000000..a0ed97f8 --- /dev/null +++ b/app/s3df/tests/test_status_adapter.py @@ -0,0 +1,307 @@ +import datetime +import json + +import httpx +import pytest + +from app.routers.status.models import ResourceType, Status +from app.s3df.status.config import ( + Backend, + Condition, + HealthCheck, + MonitoredResource, + REGISTRY, + RESOURCE_TEMPLATES, + StatusSettings, + build_registry, +) +from app.s3df.status.health_checker import HealthChecker, HealthResult, aggregate_results +from app.s3df.status.store import StatusStore +from app.s3df.status_adapter import S3DFStatusAdapter + + +EXPECTED_NAMES = [ + "SSH Bastions", + "Interactive Nodes", + "S3DF Docs", + "Batch Servers", + "Slurm", + "Monitoring", + "Coact", + "OnDemand", + "Kubernetes", + "Storage", + "DTNs", +] + +EXPECTED_IDS = [ + "s3df-ssh-bastions", + "s3df-interactive-nodes", + "s3df-docs", + "s3df-batch-servers", + "s3df-slurm", + "s3df-monitoring", + "s3df-coact", + "s3df-ondemand", + "s3df-kubernetes", + "s3df-storage", + "s3df-dtns", +] + +NOW = datetime.datetime(2026, 6, 1, 13, 0, tzinfo=datetime.timezone.utc) + + +def _settings(monkeypatch: pytest.MonkeyPatch) -> StatusSettings: + monkeypatch.delenv("S3DF_STATUS_CHECKS_JSON", raising=False) + settings = StatusSettings() + settings.prometheus_url = "https://prometheus.example" + settings.influxdb_url = "https://influx.example" + settings.poll_interval = 3600 + settings.http_timeout = 1 + return settings + + +def test_registry_matches_s3df_status_resources(): + assert [m.resource.name for m in REGISTRY] == EXPECTED_NAMES + assert [m.resource.id for m in REGISTRY] == EXPECTED_IDS + assert {m.resource.id for m in REGISTRY}.isdisjoint({"s3df-ssh-gateway", "s3df-slurmctld", "s3df-slurmdbd"}) + + checks_by_id = {m.resource.id: m.health_checks for m in REGISTRY} + assert [check.backend for check in checks_by_id["s3df-ssh-bastions"]] == [Backend.prometheus] + assert [check.backend for check in checks_by_id["s3df-slurm"]] == [Backend.influxdb, Backend.influxdb] + assert all(not checks for id_, checks in checks_by_id.items() if id_ not in {"s3df-ssh-bastions", "s3df-slurm"}) + + +def test_status_log_runtime_configuration_is_not_available(monkeypatch): + settings = _settings(monkeypatch) + + assert "status_log" not in Backend.__members__ + assert not hasattr(settings, "status_logs_base_url") + assert not hasattr(settings, "status_log_max_age") + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (200, Status.up), + (499, Status.degraded), + (500, Status.down), + (None, Status.unknown), + ], +) +def test_condition_evaluation_with_degraded_status(value, expected): + check = HealthCheck( + backend=Backend.http, + url="https://example/status", + up_when=Condition("eq", 200), + degraded_when=Condition("lt", 500), + ) + + from app.s3df.status.health_checker import evaluate + + assert evaluate(check, value) == expected + + +@pytest.mark.parametrize( + ("results", "expected"), + [ + ([], Status.unknown), + ([HealthResult(Status.unknown, None, NOW, "backend unavailable")], Status.unknown), + ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.unknown, None, NOW, "timeout")], Status.up), + ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.degraded, 0.5, NOW)], Status.degraded), + ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.down, 0.0, NOW)], Status.down), + ], +) +def test_aggregate_results(results, expected): + assert aggregate_results(results).status == expected + + +@pytest.mark.asyncio +async def test_health_checker_queries_prometheus(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/query" + assert request.url.params["query"] == "up" + return httpx.Response(200, json={"status": "success", "data": {"result": [{"value": [0, "1"]}]}}) + + settings = _settings(monkeypatch) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await HealthChecker(settings, client).check( + (HealthCheck(backend=Backend.prometheus, query="up", up_when=Condition("eq", 1)),) + ) + + assert result.status == Status.up + + +@pytest.mark.asyncio +async def test_health_checker_queries_influxdb(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/query" + assert request.url.params["db"] == "telegraf" + assert request.url.params["q"] == "SELECT mean(status_code)" + return httpx.Response(200, json={"results": [{"series": [{"values": [["2026-06-01T12:00:00Z", 0.5]]}]}]}) + + settings = _settings(monkeypatch) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await HealthChecker(settings, client).check( + ( + HealthCheck( + backend=Backend.influxdb, + query="SELECT mean(status_code)", + up_when=Condition("eq", 1), + degraded_when=Condition("gte", 0.5), + ), + ) + ) + + assert result.status == Status.degraded + + +@pytest.mark.asyncio +async def test_health_checker_queries_http_status(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://docs.example/status" + assert request.headers["x-check"] == "docs" + return httpx.Response(204) + + settings = _settings(monkeypatch) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await HealthChecker(settings, client).check( + ( + HealthCheck( + backend=Backend.http, + url="https://docs.example/status", + headers={"x-check": "docs"}, + up_when=Condition("eq", 204), + ), + ) + ) + + assert result.status == Status.up + + +@pytest.mark.asyncio +async def test_health_checker_maps_backend_failures_to_unknown(monkeypatch): + settings = _settings(monkeypatch) + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(500))) as client: + result = await HealthChecker(settings, client).check( + (HealthCheck(backend=Backend.prometheus, query="up", up_when=Condition("eq", 1)),) + ) + + assert result.status == Status.unknown + assert result.error is not None + + +def test_configured_checks_are_merged_with_builtin_checks(monkeypatch): + monkeypatch.setenv( + "S3DF_STATUS_CHECKS_JSON", + json.dumps( + { + "s3df-docs": [ + { + "backend": "http", + "name": "docs-health", + "url": "https://docs.example/status", + "up_when": {"comparator": "eq", "value": 204}, + } + ], + "s3df-slurm": [ + { + "backend": "prometheus", + "name": "slurm-exporter", + "query": "slurm_up", + "up_when": {"comparator": "gte", "value": 1}, + } + ], + } + ), + ) + + registry = build_registry(StatusSettings()) + checks_by_id = {m.resource.id: m.health_checks for m in registry} + + assert checks_by_id["s3df-docs"][0].backend == Backend.http + assert checks_by_id["s3df-docs"][0].up_when == Condition("eq", 204) + assert [check.name for check in checks_by_id["s3df-slurm"]] == [ + "slurmctld-process", + "slurmdbd-process", + "slurm-exporter", + ] + + +def test_invalid_configured_check_resource_id_is_explicit(monkeypatch): + monkeypatch.setenv("S3DF_STATUS_CHECKS_JSON", json.dumps({"s3df-missing": []})) + + with pytest.raises(ValueError, match="Unknown S3DF status resource id"): + StatusSettings() + + +@pytest.mark.asyncio +async def test_adapter_returns_cached_resources_with_current_status_and_filters(monkeypatch): + observed_at = datetime.datetime(2026, 6, 1, 12, 0, tzinfo=datetime.timezone.utc) + statuses = { + "s3df-kubernetes": Status.degraded, + "s3df-storage": Status.down, + } + + monitored = [ + MonitoredResource( + resource=resource, + health_checks=(HealthCheck(backend=Backend.http, name=resource.id, url=f"https://status.example/{resource.id}"),), + ) + for resource in RESOURCE_TEMPLATES + ] + + async def fake_check(self, checks): + return HealthResult(statuses.get(checks[0].name, Status.up), None, observed_at) + + monkeypatch.setattr(HealthChecker, "check", fake_check) + + adapter = S3DFStatusAdapter(settings=_settings(monkeypatch), monitored=monitored) + try: + resources = await adapter.get_resources(0, 100) + + assert [r.name for r in resources] == EXPECTED_NAMES + assert [r.id for r in resources] == EXPECTED_IDS + assert {r.current_status for r in resources} == {Status.up, Status.degraded, Status.down} + assert all(r.last_modified == observed_at for r in resources) + assert [r.name for r in await adapter.get_resources(0, 100, current_status=Status.down)] == ["Storage"] + assert [r.name for r in await adapter.get_resources(0, 100, group="access")] == ["SSH Bastions", "OnDemand"] + assert [r.name for r in await adapter.get_resources(0, 100, resource_type=ResourceType.website)] == ["S3DF Docs", "OnDemand"] + assert len(await adapter.get_resources(0, 100, site_id="s3df")) == len(EXPECTED_NAMES) + assert (await adapter.get_resource("Storage")).id == "s3df-storage" + finally: + await adapter.aclose() + + +def test_store_incident_lifecycle_with_cached_check_results(): + resource = REGISTRY[0].resource + store = StatusStore("s3df", [resource]) + ts1 = datetime.datetime(2026, 6, 1, 12, 0, tzinfo=datetime.timezone.utc) + ts2 = datetime.datetime(2026, 6, 1, 12, 5, tzinfo=datetime.timezone.utc) + ts3 = datetime.datetime(2026, 6, 1, 12, 10, tzinfo=datetime.timezone.utc) + ts4 = datetime.datetime(2026, 6, 1, 12, 15, tzinfo=datetime.timezone.utc) + + store.record(resource.id, HealthResult(Status.up, None, ts1)) + assert len(store.events()) == 1 + assert store.incidents() == [] + + store.record(resource.id, HealthResult(Status.down, None, ts2)) + incident = store.incidents()[0] + assert incident.status == Status.down + assert incident.resolution.value == "unresolved" + assert incident.start == ts2 + assert store.events()[-1].incident_id == incident.id + + store.record(resource.id, HealthResult(Status.unknown, None, ts3)) + assert store.incidents()[0].id == incident.id + assert store.incidents()[0].end is None + assert store.events()[-1].status == Status.unknown + assert store.events()[-1].incident_id is None + + store.record(resource.id, HealthResult(Status.up, None, ts4)) + closed = store.incidents()[0] + assert closed.id == incident.id + assert closed.status == Status.up + assert closed.end == ts4 + assert closed.resolution.value == "completed" + assert store.events()[-1].incident_id == incident.id diff --git a/design-docs/s3df-status-adapter-alternatives.md b/design-docs/s3df-status-adapter-alternatives.md index 12a1f371..434fd22b 100644 --- a/design-docs/s3df-status-adapter-alternatives.md +++ b/design-docs/s3df-status-adapter-alternatives.md @@ -49,7 +49,7 @@ status and history actually live?"** Everything below is organized around that. | **A** | Background poller + in-memory store *(current/as-built)* | none (RAM read) | process memory | none | ❌ diverges ×8 | | **B** | On-demand query + TTL cache | backend on cache-miss | nowhere (current only) | none | ⚠️ current-only | | **C** | **Prometheus as source of truth** + thin TTL cache | 1–2 Prom queries / miss | Prometheus TSDB | none | ✅ all read same TSDB | -| **D** | Read existing `s3df-status` git logs | git pull (cached) | git CSV logs | none | ✅ shared clone | +| **D** | Read dashboard git logs *(rejected)* | git pull (cached) | git CSV logs | none | ✅ shared clone | | **E** | **Proxy Alertmanager** (incidents) + Prom instant (status) | Alertmanager / Prom | Alertmanager | none\* | ✅ | | **F** | Single poller → shared Redis/SQLite, workers read | cache read | shared store | Redis/SQLite | ✅ | @@ -133,19 +133,19 @@ from it instead of re-implementing a store: resolution and retention**; incident IDs must be derived deterministically (not random) to stay stable across workers and refreshes. -### D — Reuse the existing `s3df-status` git logs +### D — Reuse dashboard git logs (rejected) -`status-pusher` already writes append-only CSV health logs to -`slaclab/s3df-status` (consumed by the Fettle dashboard). The adapter could clone/ -pull that repo and parse the logs. +The dashboard pipeline writes append-only CSV health logs that are consumed by +the S3DF status site. The adapter could clone/pull that repo and parse the logs. - **Pros:** zero new backend access; reuses a pipeline that already runs; worker-consistent via a shared clone; history already persisted in git. - **Cons:** git-pull latency on refresh; a thin point-in-time `ts,status,value` data model with **no native incident semantics** (would still need a transition fold like C, but over coarser data); couples IRI to the dashboard's repo layout. -- **Verdict:** lower-fidelity than C for the same amount of fold logic; best only - if direct Prometheus/InfluxDB access from the API is not permitted. +- **Verdict:** rejected for IRI runtime use. IRI must remain decoupled from + `status-pusher`, `s3df-status`, and dashboard log repositories; direct + IRI-configured checks are the supported polling source. ### E — Proxy Alertmanager for incidents (best incident fidelity) diff --git a/design-docs/s3df-status-adapter.md b/design-docs/s3df-status-adapter.md index 427b4449..dc7e7b2d 100644 --- a/design-docs/s3df-status-adapter.md +++ b/design-docs/s3df-status-adapter.md @@ -1,586 +1,172 @@ -# S3DF Status Adapter — Technical Design Report +# S3DF Status Adapter - Technical Design -## 1. Summary +## Summary -This document describes the implementation plan for `app/s3df/status_adapter.py` — the -S3DF-specific adapter that fulfills the IRI status API contract. Today the status router -(`/status`) has no S3DF implementation; requests require `IRI_API_ADAPTER_status` be set -to the demo adapter. The goal is a production adapter that queries real S3DF monitoring -infrastructure (Prometheus, InfluxDB) and maps results into the IRI status model -(Resources, Events, Incidents). +`app/s3df/status_adapter.py` implements the IRI `/status` router for S3DF. IRI owns status collection at runtime: it periodically runs configured health checks, evaluates status-pusher-style conditions, and caches the latest status for each resource in memory. -The `status-pusher` CLI (separate repo) already queries these same backends and evaluates -health criteria — its query and evaluation logic serves as a reference for the data-plane -portion of this adapter. +The external status repositories are references only: ---- +- `status/s3df-status` defines the user-facing resource vocabulary IRI should expose. +- `status/status-pusher` demonstrates the check/evaluate model IRI now runs internally. +- `s3df-status-logs` is not read by IRI. -## 2. Architecture Context +IRI must not fetch GitHub-hosted report logs or depend on `status-pusher`/`s3df-status` at runtime. -``` - ┌──────────────────────────────────────────────────────────────────────────┐ - │ IRI Facility API (FastAPI) │ - │ │ - │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ - │ │ /facility │ │ /account │ │ /compute │ │ /status │ │ - │ │ adapter ✓ │ │ adapter ✓ │ │ adapter ✓ │ │ adapter ✗ │ │ - │ └──────────────┘ └──────────────┘ └──────────────┘ └─────┬──────┘ │ - │ │ │ - └───────────────────────────────────────────────────────────────┼─────────┘ - │ - ┌────────────────────────────────────┘ - │ implements - ▼ - ┌────────────────────────┐ - │ S3DFStatusAdapter │ - │ app/s3df/status_ │ - │ adapter.py │ - └───────────┬────────────┘ - │ - ┌────────────┼─────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌──────────────┐ ┌────────────┐ ┌────────────────┐ - │ Prometheus │ │ InfluxDB │ │ Static config │ - │ (live health)│ │ (telegraf) │ │ (resource defs)│ - └──────────────┘ └────────────┘ └────────────────┘ -``` - -The existing `status-pusher` operates *outside* IRI as a scheduled cron/k8s job that -writes results to a git repo consumed by the Fettle dashboard. This adapter *replaces* -that indirection for the IRI API by querying backends directly at request time (or via a -background polling cache). - ---- - -## 3. IRI Status API Contract - -The abstract `FacilityAdapter` (in `app/routers/status/facility_adapter.py`) requires: - -| Method | Returns | Purpose | -|------------------|--------------------------|-----------------------------------| -| `get_resources` | `list[Resource]` | Filterable list of tracked resources | -| `get_resource` | `Resource` | Single resource by id | -| `get_events` | `list[Event]` | Status-change events | -| `get_event` | `Event` | Single event by id | -| `get_incidents` | `list[Incident]` | Planned/unplanned incidents | -| `get_incident` | `Incident` | Single incident by id | - -### Key Models - -``` -Resource: - id, name, description, last_modified, site_id, group, resource_type, - current_status (up|down|degraded|unknown), capability_ids - -Event: - id, name, description, last_modified, occurred_at, status, resource_id, - incident_id (optional) - -Incident: - id, name, description, last_modified, status, start, end, type - (planned|unplanned|reservation), resolution - (unresolved|cancelled|completed|extended|pending), resource_ids, event_ids -``` - ---- - -## 4. Reference: status-pusher - -`status-pusher` is a Click CLI that: - -1. Queries a metrics backend (Prometheus or InfluxDB). -2. Applies a success-condition comparator (eq, lt, gt, etc.) against the returned value. -3. Writes a timestamped CSV line (`zulu_ts, status_string, value`) to a log file in a git - repo (`slaclab/s3df-status`). -4. Commits + pushes the change (consumed by Fettle dashboard). - -### Relevant queries (from Makefile live tests): - -| Backend | Query | Meaning | -|------------|-------------------------------------------------------------------------------------------|------------------------------| -| Prometheus | `avg(avg_over_time(nmap_port_state{service='ssh',group='s3df'}[5m]))` | SSH gateway reachability | -| InfluxDB | `SELECT mean("status_code") FROM "monit_process" WHERE ("service" = 'slurmctld' OR "service" = 'slurmdbd') AND time > now()-5m GROUP BY "service"` | Slurm daemon health | - -### Data sources at S3DF: - -- **Prometheus**: `https://prometheus.slac.stanford.edu` — live probe metrics (nmap, etc.) -- **InfluxDB**: `https://influxdb.slac.stanford.edu`, database `telegraf` — system/service - metrics via Telegraf agents (monit, slurm, storage). - ---- - -## 5. Proposed Design - -### 5.1 Resource Registry (Static Config) - -Resources are *configuration*, not dynamic discovery. We define them via a YAML/dict -config that maps S3DF infrastructure to IRI `Resource` objects. Example: - -```yaml -resources: - - id: "s3df-ssh-gateway" - name: "SSH Login Gateway" - group: "access" - resource_type: "service" - capability_ids: [] - health_check: - backend: prometheus - query: "avg(avg_over_time(nmap_port_state{service='ssh',group='s3df'}[5m]))" - condition: {op: "eq", value: 1.0} - - - id: "s3df-slurmctld" - name: "Slurm Controller" - group: "compute" - resource_type: "compute" - capability_ids: [] - health_check: - backend: influxdb - db_name: "telegraf" - query: "SELECT mean(\"status_code\") FROM \"monit_process\" WHERE \"service\" = 'slurmctld' AND time > now()-5m" - condition: {op: "eq", value: 1.0} -``` - -This maps directly to the pattern `status-pusher` uses: query + comparator → up/down. - -### 5.2 Health Evaluation (Reused from status-pusher) - -``` - ┌──────────────────────────────────────────────────────────────┐ - │ HealthChecker (internal module) │ - │ │ - │ prometheus_query(url, query) -> (epoch_ts, value) │ - │ influx_query(url, db, query) -> (epoch_ts, value) │ - │ evaluate(value, condition) -> Status (up|down|degraded) │ - └──────────────────────────────────────────────────────────────┘ -``` - -This directly reuses the logic from `status_pusher.py`: -- `prometheus_query()` → wraps `PrometheusConnect.custom_query()` -- `influx_query()` → HTTP GET to InfluxDB `/query` endpoint -- `evaluate()` → applies comparator (eq/lt/gt/gte/lte) to determine status - -### 5.3 Caching / Polling Strategy - -Querying backends on every HTTP request would be expensive and fragile. Two options: - -**Option A: Background Poller (Recommended)** -``` - ┌─────────────────────────────────────────────────────────────┐ - │ BackgroundPoller (asyncio.Task, runs at startup) │ - │ │ - │ Every N seconds (configurable, default 60): │ - │ for resource in config.resources: │ - │ result = health_check(resource) │ - │ cache[resource.id] = StatusSnapshot(ts, status, value) │ - │ if status_changed: │ - │ events.append(new Event) │ - └─────────────────────────────────────────────────────────────┘ -``` -- Cache is an in-memory dict protected by asyncio Lock -- Events are accumulated in-memory (optionally persisted to SQLite/file for restart) -- `get_resources()` reads from cache -- `current_status` on each Resource reflects the latest polled value - -**Option B: On-demand with TTL Cache** -- Simpler; queries backend on first request, caches for TTL seconds -- Risk: slow/failed backend blocks API response - -### 5.4 Events & Incidents - -Events are generated when status *transitions* occur (e.g., up→down). The poller detects -transitions by comparing the new status to the cached previous status. - -``` - Time ──────────────────────────────────────────────► - - Resource: s3df-ssh-gateway - Poll 1: status=up (no event) - Poll 2: status=up (no event) - Poll 3: status=down → Event(occurred_at=now, status=down) - → Incident(type=unplanned, start=now, status=down) - Poll 4: status=down (no event, incident still open) - Poll 5: status=up → Event(occurred_at=now, status=up) - → Incident.end=now, resolution=completed -``` - -Incidents are auto-created for unplanned outages. Planned incidents can be injected via: -- Future: a maintenance-schedule config or API -- For MVP: all incidents are `unplanned` and auto-generated - -### 5.5 Module Structure - -``` -app/s3df/ -├── status_adapter.py # S3DFStatusAdapter class -├── status/ -│ ├── __init__.py -│ ├── config.py # Resource registry + env-driven settings -│ ├── health_checker.py # Prometheus/InfluxDB query + evaluation -│ ├── poller.py # Background polling loop -│ └── store.py # In-memory event/incident store -└── ... -``` - -### 5.6 Data Flow (Request Path) - -``` - Client GET /status/resources - │ - ▼ - status.py router - │ - ▼ - S3DFStatusAdapter.get_resources(filters...) - │ - ├─► reads resource list from config (static) - ├─► enriches current_status from cache (poller-written) - ├─► applies filters (name, group, type, status, modified_since, etc.) - └─► returns paginated list -``` - -``` - Client GET /status/events?resource_id=X&from=T1&to=T2 - │ - ▼ - S3DFStatusAdapter.get_events(filters...) - │ - ├─► reads event log from in-memory store - ├─► applies filters (resource_id, status, time range, etc.) - └─► returns paginated list -``` - ---- - -## 6. Configuration (Environment Variables) - -| Variable | Purpose | Default | -|----------------------------|--------------------------------------------|------------------------------------| -| `S3DF_PROMETHEUS_URL` | Prometheus endpoint | `https://prometheus.slac.stanford.edu` | -| `S3DF_INFLUXDB_URL` | InfluxDB endpoint | `https://influxdb.slac.stanford.edu` | -| `S3DF_INFLUXDB_DB` | InfluxDB database name | `telegraf` | -| `S3DF_STATUS_POLL_INTERVAL`| Seconds between health checks | `60` | -| `S3DF_STATUS_CONFIG_PATH` | Path to resource registry YAML (optional) | built-in defaults | - ---- - -## 7. Comparison: status-pusher vs Status Adapter +## Canonical resources -| Aspect | status-pusher | S3DF Status Adapter | -|---------------------|----------------------------------|----------------------------------------| -| Runtime model | Cron job / k8s CronJob | Long-running FastAPI background task | -| Output | CSV in git repo → Fettle | In-memory cache → IRI REST API | -| Query backends | Prometheus, InfluxDB | Same (reuse query logic) | -| Evaluation logic | comparator(value, threshold) | Same (reuse evaluation logic) | -| Event tracking | None (single point-in-time) | Stateful: detects transitions | -| Incident tracking | None | Auto-created on status transitions | -| Auth | N/A (runs internally) | Unauthenticated (status is public) | +`/status/resources` returns exactly the eleven S3DF resources from `status/s3df-status/public/urls.cfg`: ---- +| IRI id | Name | Resource type | Group | +| --- | --- | --- | --- | +| `s3df-ssh-bastions` | `SSH Bastions` | `service` | `access` | +| `s3df-interactive-nodes` | `Interactive Nodes` | `compute` | `compute` | +| `s3df-docs` | `S3DF Docs` | `website` | `documentation` | +| `s3df-batch-servers` | `Batch Servers` | `compute` | `compute` | +| `s3df-slurm` | `Slurm` | `service` | `compute` | +| `s3df-monitoring` | `Monitoring` | `service` | `operations` | +| `s3df-coact` | `Coact` | `service` | `accounts` | +| `s3df-ondemand` | `OnDemand` | `website` | `access` | +| `s3df-kubernetes` | `Kubernetes` | `system` | `platform` | +| `s3df-storage` | `Storage` | `storage` | `storage` | +| `s3df-dtns` | `DTNs` | `network` | `data-transfer` | + +The old direct-check resources (`s3df-ssh-gateway`, `s3df-slurmctld`, `s3df-slurmdbd`) are not exposed as separate resources. They are implementation details of higher-level resource checks. + +## Runtime workflow + +```text +Prometheus / InfluxDB / HTTP endpoints + | + v +IRI HealthChecker + query live source + evaluate condition + aggregate checks per resource + | + v +StatusPoller + repeats every S3DF_STATUS_POLL_INTERVAL + | + v +StatusStore cache + Resource.current_status + transition Events + auto Incidents + | + v +GET /status/resources +GET /status/events +GET /status/incidents +``` + +The first `/status/...` request starts the poller lazily and awaits one initial poll. Later requests return cached statuses; they do not query monitoring backends per request. + +## Health checks + +Each resource has zero or more checks. A check produces a scalar value and an evaluation rule: -## 8. Implementation Steps +```text +observed value + up condition + optional degraded condition -> Status +``` + +Supported backends: + +| Backend | Signal | Default `up_when` for configured checks | +| --- | --- | --- | +| `prometheus` | Instant query scalar from `/api/v1/query` | `eq 1` | +| `influxdb` | Last scalar from `/query` result series | `eq 1` | +| `http` | HTTP response status code | `eq 200` | + +Built-in checks: + +- `SSH Bastions`: Prometheus query `avg( avg_over_time(nmap_port_state{service=\`ssh\`,group=\`s3df\`}[5m]) )`, `eq 1`. +- `Slurm`: aggregate InfluxDB `monit_process` checks for `slurmctld` and `slurmdbd`, each `eq 1`. + +Other resources can receive live status through IRI configuration. + +## Check aggregation + +Multiple checks for one resource are aggregated into one `Resource.current_status`: + +- no checks -> `unknown` +- all checks `unknown` -> `unknown` +- any known check `down` -> `down` +- otherwise, any known check `degraded` -> `degraded` +- otherwise, all known checks `up` -> `up` + +Unknown checks are ignored when at least one check produced a usable signal. This prevents a transient probe failure from creating a false outage if another check confirms the resource is healthy. + +## Configuring additional live checks + +Use `S3DF_STATUS_CHECKS_JSON` to attach IRI-owned checks to any canonical resource id. Configured checks are appended to built-in checks. + +Example: + +```json +{ + "s3df-docs": [ + { + "backend": "http", + "name": "docs-home", + "url": "https://example.internal/docs/health", + "up_when": {"comparator": "eq", "value": 200} + } + ], + "s3df-kubernetes": [ + { + "backend": "prometheus", + "name": "kubernetes-api-ready", + "query": "max(up{job=\"kubernetes-apiservers\"})", + "up_when": {"comparator": "gte", "value": 1} + } + ], + "s3df-storage": [ + { + "backend": "influxdb", + "name": "storage-check", + "db_name": "telegraf", + "query": "SELECT mean(\"status_code\") FROM \"storage_health\" WHERE time > now()-5m", + "up_when": {"comparator": "eq", "value": 1} + } + ] +} +``` + +Invalid configuration raises an explicit startup/configuration error rather than silently dropping a check. Resources with no built-in or configured checks remain visible with `current_status=unknown`. + +## Configuration + +| Variable | Purpose | Default | +| --- | --- | --- | +| `S3DF_STATUS_CHECKS_JSON` | JSON mapping resource ids to extra health checks | `{}` | +| `S3DF_STATUS_POLL_INTERVAL` | Seconds between poll cycles | `60` | +| `S3DF_STATUS_HTTP_TIMEOUT` | Per-request timeout in seconds | `15` | +| `S3DF_SITE_ID` | Site id stamped onto returned resources | `s3df` | +| `S3DF_STATUS_TLS_VERIFY` | `true`, `false`, or CA bundle path for HTTP clients | `false` | +| `S3DF_PROMETHEUS_URL` | Prometheus endpoint | `https://prometheus.slac.stanford.edu` | +| `S3DF_INFLUXDB_URL` | InfluxDB endpoint | `https://influxdb.slac.stanford.edu` | +| `S3DF_INFLUXDB_DB` | Default InfluxDB database | `telegraf` | + +`make dev-s3df` and Docker select `app.s3df.status_adapter.S3DFStatusAdapter` via `IRI_API_ADAPTER_status`. -1. **Create `app/s3df/status/health_checker.py`** — Extract and adapt - `prometheus_query()` and `influx_query()` from `status-pusher`, adding async support - (use `httpx` instead of `requests` for non-blocking I/O). Port the condition evaluator. +## Events and incidents -2. **Create `app/s3df/status/config.py`** — Define the resource registry as a Python - dict/dataclass (with optional YAML override). Include health check definitions per - resource. +The store is transition-driven: -3. **Create `app/s3df/status/store.py`** — In-memory store for Events and Incidents with - append-only log semantics. Provides filtered query methods matching the adapter - interface. +- First observation emits a baseline event. +- A status change emits an event. +- `down` or `degraded` opens an unplanned incident if one is not already open. +- `down` <-> `degraded` updates the open incident. +- `up` closes the open incident with `resolution=completed`. +- `unknown` records an event but does not open or close incidents, because missing monitoring data is not a confirmed resource outage or recovery. -4. **Create `app/s3df/status/poller.py`** — Background asyncio task that periodically - runs health checks, updates resource status, and emits Events/Incidents on transitions. - -5. **Create `app/s3df/status_adapter.py`** — `S3DFStatusAdapter` class implementing - `app.routers.status.facility_adapter.FacilityAdapter`. Wires together config, cache, - and store. Starts poller on first access or app startup. - -6. **Register** — Set `IRI_API_ADAPTER_status=app.s3df.status_adapter.S3DFStatusAdapter` - in Dockerfile/deployment config. - -7. **Tests** — Unit tests mirroring `status-pusher/test/` patterns: mock Prometheus and - InfluxDB responses, verify status evaluation, event generation, and adapter filtering. - ---- - -## 9. Open Questions - -- **Persistence across restarts**: Should events/incidents survive process restarts? - Options: SQLite file, Redis, or accept ephemeral (events regenerate from live state). -- **Planned maintenance**: How should planned incidents be ingested? Config file, external - API call, or manual injection? -- **Fettle coexistence**: Should this adapter *replace* the status-pusher→Fettle pipeline - or run alongside it? (IRI could become the single source of truth for both.) -- **Additional resources**: What is the full set of resources to monitor? (SSH gateway, - slurmctld, slurmdbd, storage, network, web services, etc.) -- **Degraded vs Down thresholds**: Should we support a third threshold for `degraded` - status (e.g., partial failure), or only binary up/down per status-pusher's model? - ---- - -## 10. Dependencies - -- `prometheus-api-client` (already used by status-pusher) -- `httpx` (async HTTP for InfluxDB queries — preferred over sync `requests` in async app) -- No new external services required; uses existing Prometheus + InfluxDB infrastructure - ---- - -> **As-built addendum.** Sections 11–12 below describe the logic actually -> implemented in `app/s3df/status_adapter.py`. The adapter ships as a **single -> file** (registry + health checker + store + poller + adapter class), not the -> multi-module layout sketched in sections 5/8. Prometheus is queried directly -> over its HTTP API via `httpx` (the `prometheus-api-client` dependency was not -> needed). - -## 11. Status Computation (As Built) - -### 11.1 End-to-end pipeline - -A resource's `current_status` is derived once per poll cycle by turning a single -scalar metric into one of four `Status` values. The same pipeline runs for every -resource in `RESOURCE_REGISTRY`. - -``` - ResourceDef.health_check - ┌───────────────────────────────────────────────────────────────────────┐ - │ backend (prometheus|influxdb) + query + up_when [+ degraded_when] │ - └───────────────────────────────────────────────────────────────────────┘ - │ - ▼ HealthChecker.check() - ┌──────────────────┐ query backend over httpx (timeout-bounded) - │ Prometheus │──► GET /api/v1/query?query=... - │ or InfluxDB │──► GET /query?q=...&db=... - └──────────────────┘ - │ - ▼ parse a single scalar - value: float | None (None = empty result / parse miss) - │ - ▼ evaluate(check, value) - ┌───────────────────────────┐ - │ Status: │ - │ up | degraded | down | │ - │ unknown │ - └───────────────────────────┘ - │ - ▼ StatusStore.record(resource_id, HealthResult) - update current_status, emit Event on change, open/close Incident -``` - -### 11.2 Value → Status rule - -`evaluate()` applies the resource's conditions in a fixed precedence. A -`Condition` is just a comparator (`eq/ne/lt/lte/gt/gte`) applied to the observed -value against a threshold — the same success-criterion idea `status-pusher` uses. - -``` - value is None ───────────────────────────────► UNKNOWN - │ (query failed, errored, empty result, - │ or backend mapped to no scalar) - ▼ - up_when.met(value)? ── yes ──────────────────► UP - │ no - ▼ - degraded_when set AND degraded_when.met(value)? ─ yes ──► DEGRADED - │ no - ▼ - DOWN -``` - -Notes: -- **`unknown` ≠ `down`.** Any exception during the query (timeout, TLS, HTTP - error, malformed JSON) is caught and mapped to `unknown`, *not* `down`. A - monitoring-plane failure is not treated as a confirmed resource outage. -- **`degraded` is optional.** Resources that only define `up_when` collapse to a - binary up/down model (matching `status-pusher`). `degraded_when` is the hook - for a future third threshold. -- The registry currently uses `up_when = (value == 1.0)` for all resources - (nmap port state for SSH; monit `status_code` for slurmctld/slurmdbd). - -### 11.3 Transition detection & events - -The store is **change-driven**: `record()` compares the new status to the -resource's previous status and does nothing on a steady state. Events are emitted -only on a real change, which keeps the event log meaningful instead of one row -per poll. - -``` - poll cycle N: prev = store.current_status[r] - new = result.status - - prev == new ────────────────► no-op (steady state) - - prev != new ────────────────► emit Event(occurred_at = poll time, - status = new) - + drive incident state machine (11.4) - - Special case: prev is None (first ever observation) - → "baseline" Event describing the initial status - → if initial status is down/degraded, an incident is ALSO opened - (its start time is the first-observed time, not the true outage start) -``` - -Example for one resource over six polls: - -``` - poll: 1 2 3 4 5 6 - status: up up down down up unknown - event: base — E1 — E2 E3 - ▲ ▲ ▲ - │ │ └ up→unknown (incident stays open? no — - │ │ it was already closed at poll 5) - │ └ down→up (closes incident, resolution=completed) - └ up→down (opens unplanned incident) -``` - -### 11.4 Incident lifecycle (state machine) - -At most **one open incident per resource**. Incidents are auto-created as -`unplanned`. The driving signal is the resource's status transition: - -``` - ┌───────────────────────────────────────────┐ - │ │ - up / (start) │ │ - ───────────────► (NO OPEN INCIDENT) │ - │ │ │ - down|degraded │ │ down|degraded │ up - │ ▼ │ - (OPEN: unplanned, resolution=unresolved) │ - │ │ ▲ │ - down⇄degraded │ │ down|degraded (escalate: │ - (update incident │ │ update incident.status only) │ - status + mtime) │ └─────────────────────────────── │ - │ │ │ - │ │ up ─────────────────────────────────┘ - │ ▼ close: end=now, - │ (CLOSED: resolution=completed, status=up) - │ - unknown ───────┘ (incident is LEFT OPEN; only an event is recorded — - a backend failure must not auto-resolve an outage) -``` - -Concretely, in `record()`: -- **→ down/degraded:** open a new incident if none is open for the resource; - otherwise update the existing incident's `status`. Link the new event to the - incident (`event.incident_id`, `incident.event_ids`). -- **→ up:** pop and close the open incident (`end`, `resolution=completed`, - `status=up`), linking the closing event. -- **→ unknown:** record the event only; never opens or closes an incident. - -### 11.5 Polling cadence & lazy start - -IRI constructs adapters **synchronously at import time**, before the asyncio loop -exists, so the poller cannot start in `__init__`. Instead it starts lazily on the -first request, guarded by a double-checked `asyncio.Lock`. - -``` - import time first request (loop running) steady state - ───────────────► ─────────────────────────► ───────────► - - __init__: _ensure_started(): background task: - build store, async with start_lock: loop forever: - poller, lock if not started: sleep(interval) - (NO network, • create httpx.AsyncClient poll_once() - NO task) • await poll_once() ◄─ initial └ gather all - (populates store so the checks, record - first response isn't empty) - • create_task(_run()) - • started = True - ── request proceeds, reads store ── -``` - -- The **initial poll is awaited** under the lock, so the very first - `/status/...` response reflects real data rather than all-`unknown`. Its cost is - bounded by per-query `S3DF_STATUS_HTTP_TIMEOUT`, and `check()` never raises - (failures become `unknown`). -- All resource checks within a cycle run concurrently via `asyncio.gather`. -- `aclose()` cancels the loop and closes the HTTP client (for tests / future - FastAPI-lifespan wiring). - -## 12. Limitations & Known Trade-offs - -These are intentional simplifications for a first reviewable implementation. Each -has a clear upgrade path. - -### 12.1 Volatile, per-process state - -State (current status, events, incidents) lives in plain Python structures. It is -**lost on restart** and **diverges across uvicorn workers** — each worker polls -independently and mints its own incident/event UUIDs, so a client may see -different histories depending on which worker answers. - -``` - ┌── worker A ──┐ ┌── worker B ──┐ - │ store_A │ │ store_B │ - │ inc id=abc │ ≠ │ inc id=xyz │ ← same outage, different ids - └──────────────┘ └──────────────┘ - GET /status/incidents → load-balanced → answer depends on worker -``` - -Upgrade path: shared store (SQLite/Redis/Postgres) or a single dedicated poller -process writing a store the API workers only read. - -### 12.2 Single sample, no flap suppression - -Status is decided from **one** sample per cycle with no debounce/hysteresis. A -single transient bad scrape flips the resource and generates an event/incident; -the next good poll closes it. A flapping metric produces incident churn. - -``` - value: 1 1 0 1 0 1 (0 = "bad" scrape) - status: up up dn up dn up - events: — E E E E ← four events, two short incidents -``` +State is in-memory and per-process. Events and incidents are lost on restart and can diverge across multiple uvicorn workers. -Upgrade path: require N consecutive failures before opening (and N successes -before closing), or evaluate over a rolling window. - -### 12.3 `unknown` and unknown outage-start - -- A backend/query failure yields `unknown`, which deliberately does **not** close - an open incident — but a long backend outage will leave a resource pinned at - `unknown` with no incident of its own. -- When a resource is **already down at first observation**, the incident `start` - is set to *first-observed time*, which is not the true outage start (the - adapter has no history before it booted). - -### 12.4 Simplistic query → scalar reduction - -Each backend response is reduced to a single number: Prometheus -`data.result[0].value[1]`, InfluxDB `series[0].values[-1][1]`. Queries that return -multiple series / grouped results (e.g. `GROUP BY service`) only have their -**first** series considered. Health checks must be written to return a single -series. There is no "any/all/aggregate across series" policy yet. - -### 12.5 Scope gaps - -- **No planned incidents.** All incidents are `unplanned`; there is no ingestion - of maintenance windows (`planned` / `reservation`). Planned-maintenance support - needs a schedule source (config or API). -- **`site_id` is not a guaranteed cross-reference.** Resources carry - `S3DF_SITE_ID` (default `"s3df"`), but the `/facility` adapter currently mints a - random site UUID per process, so `Resource.site_uri` may not resolve to a real - site until a stable site id is wired through. -- **Resource set is minimal.** Only SSH gateway + slurmctld + slurmdbd are - registered today; the full S3DF resource catalogue (storage, network, web - services, …) still needs to be enumerated. - -### 12.6 Operational - -- **Unbounded event growth.** The event log is append-only and never trimmed; - a long-lived process accumulates events in memory. Needs retention/eviction - (or persistence with TTL). -- **First-request latency.** The first caller after startup pays the awaited - initial poll (up to `S3DF_STATUS_HTTP_TIMEOUT` per backend round). -- **TLS verification defaults off.** `S3DF_STATUS_TLS_VERIFY` defaults to `false` - (SLAC internal CA convenience); set it to `true` or a CA-bundle path in - production. -- **No automatic shutdown wiring.** `aclose()` exists but the framework does not - call it; wiring it into FastAPI's lifespan is a follow-up so the poller task and - HTTP client are closed cleanly on shutdown. +## Known trade-offs and follow-ups +- Checks for resources beyond SSH and Slurm require IRI configuration until canonical Prometheus/InfluxDB/HTTP probes are supplied. +- There is no persistence for status history, events, or incidents. +- There is no planned-maintenance ingestion yet; all generated incidents are `unplanned`. +- The event log is append-only in memory and has no retention policy. +- Direct Prometheus/InfluxDB checks reduce responses to one scalar and do not implement multi-series aggregation beyond resource-level check aggregation. +- `S3DF_SITE_ID` defaults to `s3df`, while the facility adapter may still use a different site identifier until a stable cross-router site id is wired through. diff --git a/local-template.env b/local-template.env index 68ad9b0c..cd87a9fa 100644 --- a/local-template.env +++ b/local-template.env @@ -7,3 +7,6 @@ export GLOBUS_RS_ID=ed3e577d-f7f3-4639-b96e-ff5a8445d699 export GLOBUS_RS_SECRET= export GLOBUS_RS_SCOPE_SUFFIX=iri_api + +# S3DF status adapter +export S3DF_STATUS_CHECKS_JSON='{}' From 2bc8a1ace10c07944043456a33c283634e9b45ea Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Mon, 1 Jun 2026 17:36:02 -0700 Subject: [PATCH 5/6] fetching status for all resources --- app/s3df/status/config.py | 182 +++++++++++++++++++++++--- app/s3df/status_adapter.py | 6 +- app/s3df/tests/test_status_adapter.py | 141 +++++++++++++++++++- design-docs/s3df-status-adapter.md | 18 ++- local-template.env | 2 + 5 files changed, 318 insertions(+), 31 deletions(-) diff --git a/app/s3df/status/config.py b/app/s3df/status/config.py index c4658e2f..666cb868 100644 --- a/app/s3df/status/config.py +++ b/app/s3df/status/config.py @@ -10,7 +10,9 @@ S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) S3DF_INFLUXDB_DB InfluxDB database (default telegraf) + S3DF_STATUS_CHECKS_FILE JSON file with resource health checks S3DF_STATUS_CHECKS_JSON JSON mapping resource ids to additional health checks + S3DF_STATUS_REQUIRE_FULL_COVERAGE Require every resource to have checks (default true) S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) S3DF_SITE_ID Site id for resources (default s3df) @@ -23,6 +25,7 @@ import os from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import Any, Callable from app.routers.status.models import Resource, ResourceType, Status @@ -231,16 +234,60 @@ def _template( } -def build_registry(settings: "StatusSettings | None" = None) -> list[MonitoredResource]: +def build_registry( + settings: "StatusSettings | None" = None, + *, + require_full_coverage: bool | None = None, +) -> list[MonitoredResource]: """Build monitored resources with built-in plus configured checks.""" configured = settings.resource_checks if settings is not None else {} - return [ + monitored = [ MonitoredResource( resource=resource, health_checks=BUILTIN_CHECKS.get(resource.id, ()) + configured.get(resource.id, ()), ) for resource in RESOURCE_TEMPLATES ] + _validate_unique_check_names(monitored) + + strict = require_full_coverage + if strict is None: + strict = settings.require_full_coverage if settings is not None else False + if strict: + _validate_full_coverage(monitored) + + return monitored + + +def missing_check_resources(monitored: list[MonitoredResource]) -> list[MonitoredResource]: + """Return canonical resources that have no configured health checks.""" + return [resource for resource in monitored if not resource.health_checks] + + +def _validate_full_coverage(monitored: list[MonitoredResource]) -> None: + missing = missing_check_resources(monitored) + if not missing: + return + labels = ", ".join(f"{m.resource.id} ({m.resource.name})" for m in missing) + raise ValueError( + "S3DF status resources missing health checks: " + f"{labels}. Configure checks with S3DF_STATUS_CHECKS_FILE or " + "S3DF_STATUS_CHECKS_JSON, or set S3DF_STATUS_REQUIRE_FULL_COVERAGE=false " + "only for development/test use." + ) + + +def _validate_unique_check_names(monitored: list[MonitoredResource]) -> None: + for resource in monitored: + seen: set[str] = set() + for check in resource.health_checks: + if check.name is None: + continue + if check.name in seen: + raise ValueError( + f"Duplicate S3DF status check name for {resource.resource.id}: {check.name}" + ) + seen.add(check.name) # Static default registry for tests/importers that do not need env-driven checks. @@ -254,7 +301,11 @@ def __init__(self) -> None: self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") - self.resource_checks = self._parse_resource_checks(os.getenv("S3DF_STATUS_CHECKS_JSON", "{}")) + self.require_full_coverage = self._parse_bool( + os.getenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", "true"), + "S3DF_STATUS_REQUIRE_FULL_COVERAGE", + ) + self.resource_checks = self._load_resource_checks() self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) # NOTE: the /facility adapter currently mints a random site uuid per @@ -263,6 +314,15 @@ def __init__(self) -> None: self.site_id = os.getenv("S3DF_SITE_ID", "s3df") self.tls_verify = self._parse_verify(os.getenv("S3DF_STATUS_TLS_VERIFY", "false")) + @staticmethod + def _parse_bool(raw: str, name: str) -> bool: + low = raw.strip().lower() + if low in ("true", "1", "yes", "on"): + return True + if low in ("false", "0", "no", "off", ""): + return False + raise ValueError(f"{name} must be a boolean value") + @staticmethod def _parse_verify(raw: str) -> bool | str: low = raw.strip().lower() @@ -273,26 +333,71 @@ def _parse_verify(raw: str) -> bool | str: return raw @classmethod - def _parse_resource_checks(cls, raw: str) -> dict[str, tuple[HealthCheck, ...]]: + def _load_resource_checks(cls) -> dict[str, tuple[HealthCheck, ...]]: + checks_file = os.getenv("S3DF_STATUS_CHECKS_FILE") + file_checks = cls._parse_resource_checks_file(checks_file) + json_checks = cls._parse_resource_checks( + os.getenv("S3DF_STATUS_CHECKS_JSON", "{}"), + "S3DF_STATUS_CHECKS_JSON", + ) + return cls._merge_resource_checks(file_checks, json_checks) + + @classmethod + def _parse_resource_checks_file(cls, checks_file: str | None) -> dict[str, tuple[HealthCheck, ...]]: + if not checks_file: + return {} + path = Path(checks_file) + try: + raw = path.read_text() + except OSError as exc: + raise ValueError(f"S3DF_STATUS_CHECKS_FILE could not be read: {checks_file}") from exc + return cls._parse_resource_checks(raw, f"S3DF_STATUS_CHECKS_FILE ({checks_file})") + + @staticmethod + def _merge_resource_checks( + *sources: dict[str, tuple[HealthCheck, ...]], + ) -> dict[str, tuple[HealthCheck, ...]]: + merged: dict[str, tuple[HealthCheck, ...]] = {} + for source in sources: + for resource_id, checks in source.items(): + merged[resource_id] = merged.get(resource_id, ()) + checks + return merged + + @classmethod + def _parse_resource_checks(cls, raw: str, source: str) -> dict[str, tuple[HealthCheck, ...]]: raw = raw.strip() if not raw: return {} - try: - data = json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError("S3DF_STATUS_CHECKS_JSON must be valid JSON") from exc + data = cls._loads_json_object(raw, source) if not isinstance(data, dict): - raise ValueError("S3DF_STATUS_CHECKS_JSON must be an object keyed by resource id") + raise ValueError(f"{source} must be an object keyed by resource id") parsed: dict[str, tuple[HealthCheck, ...]] = {} for resource_id, checks in data.items(): if resource_id not in RESOURCE_IDS: - raise ValueError(f"Unknown S3DF status resource id in S3DF_STATUS_CHECKS_JSON: {resource_id}") + raise ValueError(f"Unknown S3DF status resource id in {source}: {resource_id}") if not isinstance(checks, list): raise ValueError(f"Checks for {resource_id} must be a list") - parsed[resource_id] = tuple(cls._parse_check(resource_id, idx, check) for idx, check in enumerate(checks)) + parsed[resource_id] = tuple( + cls._parse_check(resource_id, idx, check) for idx, check in enumerate(checks) + ) return parsed + @staticmethod + def _loads_json_object(raw: str, source: str) -> Any: + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + data: dict[str, Any] = {} + for key, value in pairs: + if key in data: + raise ValueError(f"{source} contains duplicate key: {key}") + data[key] = value + return data + + try: + return json.loads(raw, object_pairs_hook=reject_duplicate_keys) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} must be valid JSON") from exc + @classmethod def _parse_check(cls, resource_id: str, idx: int, raw: Any) -> HealthCheck: if not isinstance(raw, dict): @@ -302,31 +407,68 @@ def _parse_check(cls, resource_id: str, idx: int, raw: Any) -> HealthCheck: except KeyError as exc: raise ValueError(f"Check {idx} for {resource_id} is missing backend") from exc except ValueError as exc: - raise ValueError(f"Check {idx} for {resource_id} has unsupported backend: {raw.get('backend')}") from exc + raise ValueError( + f"Check {idx} for {resource_id} has unsupported backend: {raw.get('backend')}" + ) from exc - up_when = cls._parse_condition(raw.get("up_when"), default=Condition("eq", 200.0 if backend == Backend.http else 1.0)) + up_when = cls._parse_condition( + raw.get("up_when"), + default=Condition("eq", 200.0 if backend == Backend.http else 1.0), + ) degraded_when = cls._parse_condition(raw.get("degraded_when"), default=None) + name = cls._optional_str( + raw.get("name"), + f"Check {idx} for {resource_id} has invalid name", + ) + query = cls._optional_str( + raw.get("query"), + f"Check {idx} for {resource_id} has invalid query", + ) + db_name = cls._optional_str( + raw.get("db_name"), + f"Check {idx} for {resource_id} has invalid db_name", + ) + url = cls._optional_str( + raw.get("url"), + f"Check {idx} for {resource_id} has invalid url", + ) + method = raw.get("method", "GET") + if not isinstance(method, str) or not method.strip(): + raise ValueError(f"Check {idx} for {resource_id} has invalid method") + follow_redirects = raw.get("follow_redirects", True) + if not isinstance(follow_redirects, bool): + raise ValueError(f"Check {idx} for {resource_id} has invalid follow_redirects") headers = raw.get("headers", {}) if headers is None: headers = {} - if not isinstance(headers, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in headers.items()): + if not isinstance(headers, dict) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in headers.items() + ): raise ValueError(f"Check {idx} for {resource_id} has invalid headers") check = HealthCheck( backend=backend, - name=raw.get("name"), - query=raw.get("query"), + name=name, + query=query, up_when=up_when, - db_name=raw.get("db_name"), + db_name=db_name, degraded_when=degraded_when, - url=raw.get("url"), - method=str(raw.get("method", "GET")).upper(), + url=url, + method=method.strip().upper(), headers=headers, - follow_redirects=bool(raw.get("follow_redirects", True)), + follow_redirects=follow_redirects, ) cls._validate_check(resource_id, idx, check) return check + @staticmethod + def _optional_str(value: Any, error: str) -> str | None: + if value is None: + return None + if not isinstance(value, str) or not value.strip(): + raise ValueError(error) + return value + @staticmethod def _parse_condition(raw: Any, default: Condition | None) -> Condition | None: if raw is None: diff --git a/app/s3df/status_adapter.py b/app/s3df/status_adapter.py index cc5d5a18..647e4f54 100644 --- a/app/s3df/status_adapter.py +++ b/app/s3df/status_adapter.py @@ -7,8 +7,10 @@ Design (see design-docs/s3df-status-adapter.md): * Resources are STATIC config (``app.s3df.status.config``). Each pairs an IRI - ``Resource`` template with zero or more health checks. Built-in checks are - merged with optional ``S3DF_STATUS_CHECKS_JSON`` checks at adapter startup. + ``Resource`` template with one or more health checks. Built-in checks are + merged with optional ``S3DF_STATUS_CHECKS_FILE`` and + ``S3DF_STATUS_CHECKS_JSON`` checks at adapter startup; full coverage is + required by default. * A background poller (``app.s3df.status.poller``) runs every ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries each resource's configured sources, maps the aggregate result to a Status (up/down/degraded/unknown), diff --git a/app/s3df/tests/test_status_adapter.py b/app/s3df/tests/test_status_adapter.py index a0ed97f8..204a5f9a 100644 --- a/app/s3df/tests/test_status_adapter.py +++ b/app/s3df/tests/test_status_adapter.py @@ -14,6 +14,7 @@ RESOURCE_TEMPLATES, StatusSettings, build_registry, + missing_check_resources, ) from app.s3df.status.health_checker import HealthChecker, HealthResult, aggregate_results from app.s3df.status.store import StatusStore @@ -51,8 +52,30 @@ NOW = datetime.datetime(2026, 6, 1, 13, 0, tzinfo=datetime.timezone.utc) -def _settings(monkeypatch: pytest.MonkeyPatch) -> StatusSettings: +def _clear_status_check_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("S3DF_STATUS_CHECKS_FILE", raising=False) monkeypatch.delenv("S3DF_STATUS_CHECKS_JSON", raising=False) + monkeypatch.delenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", raising=False) + + +def _full_coverage_config() -> dict[str, list[dict[str, object]]]: + builtin_ids = {"s3df-ssh-bastions", "s3df-slurm"} + return { + resource.id: [ + { + "backend": "http", + "name": f"{resource.id}-status", + "url": f"https://status.example/{resource.id}", + "up_when": {"comparator": "eq", "value": 200}, + } + ] + for resource in RESOURCE_TEMPLATES + if resource.id not in builtin_ids + } + + +def _settings(monkeypatch: pytest.MonkeyPatch) -> StatusSettings: + _clear_status_check_env(monkeypatch) settings = StatusSettings() settings.prometheus_url = "https://prometheus.example" settings.influxdb_url = "https://influx.example" @@ -72,6 +95,29 @@ def test_registry_matches_s3df_status_resources(): assert all(not checks for id_, checks in checks_by_id.items() if id_ not in {"s3df-ssh-bastions", "s3df-slurm"}) +def test_build_registry_requires_full_coverage_by_default(monkeypatch): + _clear_status_check_env(monkeypatch) + + with pytest.raises(ValueError, match="S3DF status resources missing health checks") as exc_info: + build_registry(StatusSettings()) + + assert "s3df-interactive-nodes" in str(exc_info.value) + assert "S3DF_STATUS_CHECKS_FILE" in str(exc_info.value) + + +def test_build_registry_allows_missing_checks_only_when_explicitly_non_strict(monkeypatch): + _clear_status_check_env(monkeypatch) + monkeypatch.setenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", "false") + + registry = build_registry(StatusSettings()) + + assert [m.resource.id for m in missing_check_resources(registry)] == [ + resource.id + for resource in RESOURCE_TEMPLATES + if resource.id not in {"s3df-ssh-bastions", "s3df-slurm"} + ] + + def test_status_log_runtime_configuration_is_not_available(monkeypatch): settings = _settings(monkeypatch) @@ -192,6 +238,7 @@ async def test_health_checker_maps_backend_failures_to_unknown(monkeypatch): def test_configured_checks_are_merged_with_builtin_checks(monkeypatch): + _clear_status_check_env(monkeypatch) monkeypatch.setenv( "S3DF_STATUS_CHECKS_JSON", json.dumps( @@ -216,7 +263,7 @@ def test_configured_checks_are_merged_with_builtin_checks(monkeypatch): ), ) - registry = build_registry(StatusSettings()) + registry = build_registry(StatusSettings(), require_full_coverage=False) checks_by_id = {m.resource.id: m.health_checks for m in registry} assert checks_by_id["s3df-docs"][0].backend == Backend.http @@ -228,13 +275,103 @@ def test_configured_checks_are_merged_with_builtin_checks(monkeypatch): ] +def test_check_file_can_provide_full_coverage(monkeypatch, tmp_path): + _clear_status_check_env(monkeypatch) + check_file = tmp_path / "s3df-status-checks.json" + check_file.write_text(json.dumps(_full_coverage_config())) + monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", str(check_file)) + + registry = build_registry(StatusSettings()) + checks_by_id = {m.resource.id: m.health_checks for m in registry} + + assert not missing_check_resources(registry) + assert checks_by_id["s3df-docs"][0].backend == Backend.http + assert checks_by_id["s3df-docs"][0].url == "https://status.example/s3df-docs" + + +def test_check_file_and_inline_json_are_merged(monkeypatch, tmp_path): + _clear_status_check_env(monkeypatch) + check_file = tmp_path / "s3df-status-checks.json" + check_file.write_text( + json.dumps( + { + "s3df-docs": [ + { + "backend": "http", + "name": "docs-file", + "url": "https://docs.example/from-file", + } + ] + } + ) + ) + monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", str(check_file)) + monkeypatch.setenv( + "S3DF_STATUS_CHECKS_JSON", + json.dumps( + { + "s3df-docs": [ + { + "backend": "http", + "name": "docs-inline", + "url": "https://docs.example/from-inline", + } + ] + } + ), + ) + + registry = build_registry(StatusSettings(), require_full_coverage=False) + docs_checks = {m.resource.id: m.health_checks for m in registry}["s3df-docs"] + + assert [check.name for check in docs_checks] == ["docs-file", "docs-inline"] + + def test_invalid_configured_check_resource_id_is_explicit(monkeypatch): + _clear_status_check_env(monkeypatch) monkeypatch.setenv("S3DF_STATUS_CHECKS_JSON", json.dumps({"s3df-missing": []})) with pytest.raises(ValueError, match="Unknown S3DF status resource id"): StatusSettings() +def test_duplicate_json_keys_are_explicit(monkeypatch): + _clear_status_check_env(monkeypatch) + monkeypatch.setenv("S3DF_STATUS_CHECKS_JSON", '{"s3df-docs": [], "s3df-docs": []}') + + with pytest.raises(ValueError, match="duplicate key: s3df-docs"): + StatusSettings() + + +def test_duplicate_check_names_are_explicit(monkeypatch): + _clear_status_check_env(monkeypatch) + monkeypatch.setenv( + "S3DF_STATUS_CHECKS_JSON", + json.dumps( + { + "s3df-slurm": [ + { + "backend": "prometheus", + "name": "slurmctld-process", + "query": "slurm_up", + } + ] + } + ), + ) + + with pytest.raises(ValueError, match="Duplicate S3DF status check name"): + build_registry(StatusSettings(), require_full_coverage=False) + + +def test_missing_check_file_is_explicit(monkeypatch): + _clear_status_check_env(monkeypatch) + monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", "/does/not/exist.json") + + with pytest.raises(ValueError, match="S3DF_STATUS_CHECKS_FILE could not be read"): + StatusSettings() + + @pytest.mark.asyncio async def test_adapter_returns_cached_resources_with_current_status_and_filters(monkeypatch): observed_at = datetime.datetime(2026, 6, 1, 12, 0, tzinfo=datetime.timezone.utc) diff --git a/design-docs/s3df-status-adapter.md b/design-docs/s3df-status-adapter.md index dc7e7b2d..79ec6c1c 100644 --- a/design-docs/s3df-status-adapter.md +++ b/design-docs/s3df-status-adapter.md @@ -63,7 +63,7 @@ The first `/status/...` request starts the poller lazily and awaits one initial ## Health checks -Each resource has zero or more checks. A check produces a scalar value and an evaluation rule: +Each resource must have one or more checks in the default production configuration. A check produces a scalar value and an evaluation rule: ```text observed value + up condition + optional degraded condition -> Status @@ -82,13 +82,13 @@ Built-in checks: - `SSH Bastions`: Prometheus query `avg( avg_over_time(nmap_port_state{service=\`ssh\`,group=\`s3df\`}[5m]) )`, `eq 1`. - `Slurm`: aggregate InfluxDB `monit_process` checks for `slurmctld` and `slurmdbd`, each `eq 1`. -Other resources can receive live status through IRI configuration. +The remaining resources receive live status through IRI configuration. If strict coverage is enabled, startup fails until each canonical resource has at least one built-in or configured check. ## Check aggregation Multiple checks for one resource are aggregated into one `Resource.current_status`: -- no checks -> `unknown` +- no checks -> configuration error by default; `unknown` only in explicit non-strict development/test mode - all checks `unknown` -> `unknown` - any known check `down` -> `down` - otherwise, any known check `degraded` -> `degraded` @@ -96,9 +96,11 @@ Multiple checks for one resource are aggregated into one `Resource.current_statu Unknown checks are ignored when at least one check produced a usable signal. This prevents a transient probe failure from creating a false outage if another check confirms the resource is healthy. -## Configuring additional live checks +## Configuring live checks -Use `S3DF_STATUS_CHECKS_JSON` to attach IRI-owned checks to any canonical resource id. Configured checks are appended to built-in checks. +Use `S3DF_STATUS_CHECKS_FILE` and/or `S3DF_STATUS_CHECKS_JSON` to attach IRI-owned checks to any canonical resource id. Configured checks are appended to built-in checks. File config is read first, then inline JSON is appended. + +Strict full coverage is enabled by default through `S3DF_STATUS_REQUIRE_FULL_COVERAGE=true`. In that mode, IRI fails startup/configuration if any canonical resource has no checks. Setting `S3DF_STATUS_REQUIRE_FULL_COVERAGE=false` is intended only for development and tests where missing probes should be shown as `unknown`. Example: @@ -132,13 +134,15 @@ Example: } ``` -Invalid configuration raises an explicit startup/configuration error rather than silently dropping a check. Resources with no built-in or configured checks remain visible with `current_status=unknown`. +Invalid configuration raises an explicit startup/configuration error rather than silently dropping a check. In strict mode, resources with no built-in or configured checks also raise an explicit error rather than being served as hard-coded `unknown`. ## Configuration | Variable | Purpose | Default | | --- | --- | --- | -| `S3DF_STATUS_CHECKS_JSON` | JSON mapping resource ids to extra health checks | `{}` | +| `S3DF_STATUS_REQUIRE_FULL_COVERAGE` | Require every canonical resource to have at least one check | `true` | +| `S3DF_STATUS_CHECKS_FILE` | JSON file mapping resource ids to extra health checks | unset | +| `S3DF_STATUS_CHECKS_JSON` | Inline JSON mapping resource ids to extra health checks | `{}` | | `S3DF_STATUS_POLL_INTERVAL` | Seconds between poll cycles | `60` | | `S3DF_STATUS_HTTP_TIMEOUT` | Per-request timeout in seconds | `15` | | `S3DF_SITE_ID` | Site id stamped onto returned resources | `s3df` | diff --git a/local-template.env b/local-template.env index cd87a9fa..79e2a0df 100644 --- a/local-template.env +++ b/local-template.env @@ -9,4 +9,6 @@ export GLOBUS_RS_SECRET= Date: Fri, 5 Jun 2026 14:48:42 -0700 Subject: [PATCH 6/6] status and filesystem adapters point to microservices --- Dockerfile | 2 + app/s3df/clients/__init__.py | 18 + app/s3df/clients/fs_facade.py | 170 +++++++ app/s3df/clients/s3df_status_api.py | 136 ++++++ app/s3df/config.py | 9 + app/s3df/filesystem_adapter.py | 253 ++++++++-- app/s3df/status/__init__.py | 41 -- app/s3df/status/config.py | 490 -------------------- app/s3df/status/health_checker.py | 147 ------ app/s3df/status/poller.py | 79 ---- app/s3df/status/store.py | 147 ------ app/s3df/status_adapter.py | 280 +++++++---- app/s3df/status_registry.py | 132 ++++++ app/s3df/tests/test_filesystem_adapter.py | 170 +++++++ app/s3df/tests/test_status_adapter.py | 541 ++++++---------------- app/s3df/tests/test_status_registry.py | 55 +++ app/s3df/tests/test_user_lookup.py | 50 ++ 17 files changed, 1294 insertions(+), 1426 deletions(-) create mode 100644 app/s3df/clients/fs_facade.py create mode 100644 app/s3df/clients/s3df_status_api.py delete mode 100644 app/s3df/status/__init__.py delete mode 100644 app/s3df/status/config.py delete mode 100644 app/s3df/status/health_checker.py delete mode 100644 app/s3df/status/poller.py delete mode 100644 app/s3df/status/store.py create mode 100644 app/s3df/status_registry.py create mode 100644 app/s3df/tests/test_filesystem_adapter.py create mode 100644 app/s3df/tests/test_status_registry.py create mode 100644 app/s3df/tests/test_user_lookup.py diff --git a/Dockerfile b/Dockerfile index 266fef42..17d6f2dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN pip install -U pip wheel setuptools uv && \ ENV IRI_API_ADAPTER_account="app.s3df.account_adapter.S3DFAccountAdapter" ENV IRI_API_ADAPTER_status="app.s3df.status_adapter.S3DFStatusAdapter" +ENV IRI_API_ADAPTER_compute="app.s3df.compute_adapter.SLACComputeAdapter" +ENV IRI_API_ADAPTER_filesystem="app.s3df.filesystem_adapter.S3DFFilesystemAdapter" ENV IRI_SHOW_MISSING_ROUTES="true" ENV DEX_JWKS_URL="https://dex.slac.stanford.edu/keys" ENV DEX_ISSUER="https://dex.slac.stanford.edu" diff --git a/app/s3df/clients/__init__.py b/app/s3df/clients/__init__.py index ace42855..305326a0 100644 --- a/app/s3df/clients/__init__.py +++ b/app/s3df/clients/__init__.py @@ -5,11 +5,29 @@ """ from app.s3df.clients.coact import CoactClient, get_coact_client +from app.s3df.clients.fs_facade import ( + FsFacadeClient, + FsFacadeError, + FsFacadeTimeout, + get_fs_facade_client, +) +from app.s3df.clients.s3df_status_api import ( + S3DFStatusApiClient, + S3DFStatusApiError, + get_s3df_status_api_client, +) from app.s3df.clients.user_lookup import UserLookupClient, get_user_lookup_client __all__ = [ "CoactClient", "get_coact_client", + "FsFacadeClient", + "FsFacadeError", + "FsFacadeTimeout", + "get_fs_facade_client", + "S3DFStatusApiClient", + "S3DFStatusApiError", + "get_s3df_status_api_client", "UserLookupClient", "get_user_lookup_client", ] diff --git a/app/s3df/clients/fs_facade.py b/app/s3df/clients/fs_facade.py new file mode 100644 index 00000000..ec5067b9 --- /dev/null +++ b/app/s3df/clients/fs_facade.py @@ -0,0 +1,170 @@ +""" +fs-facade-service Client + +Async client for the filesystem facade microservice. Submits a filesystem +operation, polls the task endpoint until terminal state, and returns the +parsed JSON ``result`` payload. + +The microservice exposes a task-queue API: + * ``POST/GET/PUT/DELETE /filesystem/{op}`` -> returns a bare ``task_id`` string + * ``GET /task/{task_id}`` -> returns ``{"output": Task{id,status,result,command}}`` + where ``result`` is itself a JSON-encoded string (set by the dispatcher). + +See: ``fs-facade-service/app/controllers/{filesystem_controller,task_controller}.py``. +""" + +import asyncio +import json +import logging +from typing import Any, Optional + +import httpx + +from app.s3df.config import settings + +LOG = logging.getLogger(__name__) + + +class FsFacadeError(Exception): + """Raised when fs-facade returns an error or a task fails.""" + + +class FsFacadeTimeout(Exception): + """Raised when polling exceeds the configured timeout.""" + + +_TERMINAL = {"completed", "failed", "canceled"} + + +class FsFacadeClient: + """Async client for fs-facade-service.""" + + def __init__( + self, + base_url: str | None = None, + poll_interval: float | None = None, + timeout: float | None = None, + ): + self.base_url = (base_url or settings.fs_facade_url).rstrip("/") + self.poll_interval = poll_interval if poll_interval is not None else settings.fs_facade_poll_interval + self.timeout = timeout if timeout is not None else settings.fs_facade_timeout + self._client: httpx.AsyncClient | None = None + LOG.info(f"Initialized FsFacadeClient for endpoint: {self.base_url}") + + def _get_client(self) -> httpx.AsyncClient: + if self._client is None: + self._client = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) + return self._client + + async def aclose(self) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + async def _request_task_id(self, method: str, path: str, **kwargs) -> str: + """Issue an HTTP request that returns a bare task_id string.""" + client = self._get_client() + try: + resp = await client.request(method, path, **kwargs) + except httpx.HTTPError as exc: + raise FsFacadeError(f"fs-facade transport error: {exc}") from exc + if resp.status_code >= 400: + raise FsFacadeError( + f"fs-facade {method} {path} -> {resp.status_code}: {resp.text}" + ) + data = resp.json() + # The controllers return either a bare string (response_model=str) or + # `{"task_id": "..."}` (the create_task endpoint). Accept both shapes. + if isinstance(data, str): + return data + if isinstance(data, dict) and "task_id" in data: + return data["task_id"] + raise FsFacadeError(f"fs-facade returned unexpected payload: {data!r}") + + async def get_task(self, task_id: str) -> dict: + """Fetch the current task record.""" + client = self._get_client() + try: + resp = await client.get(f"/task/{task_id}") + except httpx.HTTPError as exc: + raise FsFacadeError(f"fs-facade transport error: {exc}") from exc + if resp.status_code == 404: + raise FsFacadeError(f"fs-facade task not found: {task_id}") + if resp.status_code >= 400: + raise FsFacadeError( + f"fs-facade GET /task/{task_id} -> {resp.status_code}: {resp.text}" + ) + body = resp.json() + # Wrapped in `{"output": Task{...}}` per the task_controller. + return body.get("output", body) if isinstance(body, dict) else body + + async def wait(self, task_id: str, timeout: float | None = None) -> dict: + """Poll until the task reaches a terminal state. Returns the Task dict.""" + deadline_left = timeout if timeout is not None else self.timeout + elapsed = 0.0 + while True: + task = await self.get_task(task_id) + status = task.get("status") + if status in _TERMINAL: + return task + if elapsed >= deadline_left: + raise FsFacadeTimeout( + f"Timed out waiting for fs-facade task {task_id} (status={status})" + ) + await asyncio.sleep(self.poll_interval) + elapsed += self.poll_interval + + async def call( + self, + method: str, + path: str, + *, + params: dict | None = None, + json_body: Any | None = None, + files: dict | None = None, + data: dict | None = None, + timeout: float | None = None, + ) -> Any: + """Submit an operation, wait for terminal state, and return the parsed result. + + On success returns the JSON-decoded ``result`` (or the raw string when + the dispatcher returned a non-JSON payload). On failure raises + ``FsFacadeError``. + """ + kwargs: dict[str, Any] = {} + if params is not None: + kwargs["params"] = params + if json_body is not None: + kwargs["json"] = json_body + if files is not None: + kwargs["files"] = files + if data is not None: + kwargs["data"] = data + + task_id = await self._request_task_id(method, path, **kwargs) + task = await self.wait(task_id, timeout=timeout) + + status = task.get("status") + result = task.get("result") + if status != "completed": + raise FsFacadeError( + f"fs-facade task {task_id} ended with status={status}: {result}" + ) + + if isinstance(result, str): + try: + return json.loads(result) + except (TypeError, ValueError): + return result + return result + + +_default_client: Optional[FsFacadeClient] = None + + +def get_fs_facade_client() -> FsFacadeClient: + """Get or create the singleton FsFacadeClient instance.""" + global _default_client + if _default_client is None: + _default_client = FsFacadeClient() + return _default_client diff --git a/app/s3df/clients/s3df_status_api.py b/app/s3df/clients/s3df_status_api.py new file mode 100644 index 00000000..c4c3a45a --- /dev/null +++ b/app/s3df/clients/s3df_status_api.py @@ -0,0 +1,136 @@ +""" +s3df-status-api Client + +Async client for the S3DF status microservice. Returns raw JSON dicts to +keep the IRI status adapter responsible for shape conversion. + +Endpoints (see ``s3df-status-api/app/controllers/status.py``): + * ``GET /api/v1/resources`` -> list[ResourceStatus] + * ``GET /api/v1/resources/{resource_id}`` -> ResourceStatus + * ``GET /api/v1/events?resource_id&since&limit`` -> list[StatusEvent] + * ``GET /api/v1/incidents?resource_id&state`` -> list[Incident] + +When events/incidents are disabled in the upstream service the relevant +endpoints return 404; we map that to an empty list so callers do not need +to handle it specially. +""" + +import datetime +import logging +from typing import Optional + +import httpx + +from app.s3df.config import settings + +LOG = logging.getLogger(__name__) + + +class S3DFStatusApiError(Exception): + """Raised when the upstream returns an unexpected error.""" + + +class S3DFStatusApiClient: + """Async client for s3df-status-api.""" + + def __init__(self, base_url: str | None = None, timeout: float | None = None): + self.base_url = (base_url or settings.s3df_status_api_url).rstrip("/") + self.timeout = timeout if timeout is not None else settings.s3df_status_api_timeout + self._client: httpx.AsyncClient | None = None + LOG.info(f"Initialized S3DFStatusApiClient for endpoint: {self.base_url}") + + def _get_client(self) -> httpx.AsyncClient: + if self._client is None: + self._client = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) + return self._client + + async def aclose(self) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + async def _get(self, path: str, *, params: dict | None = None) -> httpx.Response: + client = self._get_client() + try: + return await client.get(path, params=params) + except httpx.HTTPError as exc: + raise S3DFStatusApiError(f"s3df-status-api transport error: {exc}") from exc + + async def list_resource_statuses( + self, + group: str | None = None, + status: str | None = None, + ) -> list[dict]: + params: dict = {} + if group is not None: + params["group"] = group + if status is not None: + params["status"] = status + resp = await self._get("/api/v1/resources", params=params or None) + if resp.status_code >= 400: + raise S3DFStatusApiError( + f"GET /resources -> {resp.status_code}: {resp.text}" + ) + return resp.json() + + async def get_resource_status(self, resource_id: str) -> dict | None: + resp = await self._get(f"/api/v1/resources/{resource_id}") + if resp.status_code == 404: + return None + if resp.status_code >= 400: + raise S3DFStatusApiError( + f"GET /resources/{resource_id} -> {resp.status_code}: {resp.text}" + ) + return resp.json() + + async def list_events( + self, + resource_id: str | None = None, + since: datetime.datetime | None = None, + limit: int = 1000, + ) -> list[dict]: + params: dict = {"limit": limit} + if resource_id is not None: + params["resource_id"] = resource_id + if since is not None: + params["since"] = since.isoformat() + resp = await self._get("/api/v1/events", params=params) + if resp.status_code == 404: + # events feature disabled upstream + return [] + if resp.status_code >= 400: + raise S3DFStatusApiError( + f"GET /events -> {resp.status_code}: {resp.text}" + ) + return resp.json() + + async def list_incidents( + self, + resource_id: str | None = None, + state: str | None = None, + ) -> list[dict]: + params: dict = {} + if resource_id is not None: + params["resource_id"] = resource_id + if state is not None: + params["state"] = state + resp = await self._get("/api/v1/incidents", params=params or None) + if resp.status_code == 404: + # incidents feature disabled upstream + return [] + if resp.status_code >= 400: + raise S3DFStatusApiError( + f"GET /incidents -> {resp.status_code}: {resp.text}" + ) + return resp.json() + + +_default_client: Optional[S3DFStatusApiClient] = None + + +def get_s3df_status_api_client() -> S3DFStatusApiClient: + """Get or create the singleton S3DFStatusApiClient instance.""" + global _default_client + if _default_client is None: + _default_client = S3DFStatusApiClient() + return _default_client diff --git a/app/s3df/config.py b/app/s3df/config.py index 207052e6..a01e58d6 100644 --- a/app/s3df/config.py +++ b/app/s3df/config.py @@ -40,5 +40,14 @@ def __init__(self): # user-lookup service (direct LDAP/POSIX identity queries) self.user_lookup_url = os.getenv("USER_LOOKUP_URL", "") + # fs-facade-service (filesystem operations microservice) + self.fs_facade_url = os.getenv("FS_FACADE_URL", "http://fs-facade-service:8100") + self.fs_facade_poll_interval = float(os.getenv("FS_FACADE_POLL_INTERVAL", "0.25")) + self.fs_facade_timeout = float(os.getenv("FS_FACADE_TIMEOUT", "60")) + + # s3df-status-api (status microservice) + self.s3df_status_api_url = os.getenv("S3DF_STATUS_API_URL", "http://s3df-status-api:8080") + self.s3df_status_api_timeout = float(os.getenv("S3DF_STATUS_API_TIMEOUT", "10")) + settings = S3DFSettings() diff --git a/app/s3df/filesystem_adapter.py b/app/s3df/filesystem_adapter.py index d3db4c08..92ecc801 100644 --- a/app/s3df/filesystem_adapter.py +++ b/app/s3df/filesystem_adapter.py @@ -1,26 +1,70 @@ """ S3DF Filesystem Adapter -Thin proxy adapter for filesystem operations. Authenticates via Dex JWT, -enriches with POSIX identity from user-lookup, and will forward requests -to the filesystem microservice (same endpoints, same data model). +Forwards every IRI filesystem operation to the ``fs-facade-service`` +microservice, polls the task endpoint until it reaches a terminal state, +and converts the result into the matching IRI response model. + +Authentication: Dex JWT verification (mixin); POSIX identity is fetched +from user-lookup and attached to the IRI ``User`` object. The user +identity is not yet forwarded to fs-facade — see TODO below. + +See ``fs-facade-service/app/controllers/filesystem_controller.py`` for +the upstream wire format. """ +import base64 import logging from fastapi import HTTPException from app.types.user import User from app.s3df.auth.authenticated_adapter import S3DFAuthenticatedAdapter -from app.s3df.clients import get_user_lookup_client +from app.s3df.clients import ( + FsFacadeError, + FsFacadeTimeout, + get_fs_facade_client, + get_user_lookup_client, +) from app.routers.filesystem import facility_adapter, models from app.routers.status import models as status_models LOG = logging.getLogger(__name__) +async def _fs_call(method: str, path: str, **kwargs): + """Submit an op via fs-facade and translate transport/timeout errors to HTTP.""" + client = get_fs_facade_client() + try: + return await client.call(method, path, **kwargs) + except FsFacadeTimeout as exc: + LOG.warning(f"fs-facade timeout on {method} {path}: {exc}") + raise HTTPException(status_code=504, detail=f"fs-facade timeout: {exc}") from exc + except FsFacadeError as exc: + LOG.error(f"fs-facade error on {method} {path}: {exc}") + raise HTTPException(status_code=502, detail=f"fs-facade error: {exc}") from exc + + +def _file_model(payload) -> models.File: + """Convert a fs-facade File JSON dict into the IRI File model.""" + if payload is None: + raise HTTPException(status_code=502, detail="fs-facade returned no file payload") + return models.File.model_validate(payload) + + +def _content_payload(text: str, *, content_type: models.ContentUnit, offset: int = 0) -> models.FileContent: + """Wrap raw text from head/tail/view into the IRI FileContent shape.""" + text = text or "" + return models.FileContent( + content=text, + content_type=content_type, + start_position=offset, + end_position=offset + len(text), + ) + + class S3DFFilesystemAdapter(S3DFAuthenticatedAdapter, facility_adapter.FacilityAdapter): - """Filesystem adapter that enforces POSIX identity via user-lookup.""" + """Filesystem adapter that forwards operations to fs-facade-service.""" async def get_user(self, user_id: str, api_key: str, client_ip: str | None, globus_introspect: dict | None = None) -> User: try: @@ -38,58 +82,209 @@ async def get_user(self, user_id: str, api_key: str, client_ip: str | None, glob client_ip=client_ip, ) - # --- Filesystem operations (501 until microservice is connected) --- + # --- Permissions / ownership ------------------------------------------ async def chmod(self, resource: status_models.Resource, user: User, request_model: models.PutFileChmodRequest) -> models.PutFileChmodResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "PUT", "/filesystem/chmod", + json_body={"path": request_model.path, "mode": request_model.mode}, + ) + return models.PutFileChmodResponse(output=_file_model(result)) async def chown(self, resource: status_models.Resource, user: User, request_model: models.PutFileChownRequest) -> models.PutFileChownResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "PUT", "/filesystem/chown", + json_body={ + "path": request_model.path, + "owner": request_model.owner, + "group": request_model.group, + }, + ) + return models.PutFileChownResponse(output=_file_model(result)) - async def ls(self, resource: status_models.Resource, user: User, path: str, show_hidden: bool, numeric_uid: bool, recursive: bool, dereference: bool) -> models.GetDirectoryLsResponse: - raise HTTPException(status_code=501, detail="Not implemented") + # --- Directory listing ------------------------------------------------ - async def head(self, resource: status_models.Resource, user: User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> models.GetFileHeadResponse: - raise HTTPException(status_code=501, detail="Not implemented") + async def ls( + self, + resource: status_models.Resource, + user: User, + path: str, + show_hidden: bool, + numeric_uid: bool, + recursive: bool, + dereference: bool, + ) -> models.GetDirectoryLsResponse: + # NOTE: fs-facade-service currently honours only `show_hidden`. The + # other flags (numeric_uid, recursive, dereference) are silently + # ignored. Track adding upstream support if/when needed. + result = await _fs_call( + "GET", "/filesystem/ls", + params={"path": path, "show_hidden": str(show_hidden).lower()}, + ) + if not isinstance(result, list): + raise HTTPException(status_code=502, detail=f"fs-facade ls returned {type(result).__name__}") + return models.GetDirectoryLsResponse(output=[models.File.model_validate(f) for f in result]) - async def tail(self, resource: status_models.Resource, user: User, path: str, file_bytes: int | None, lines: int | None, skip_heading: bool) -> models.GetFileTailResponse: - raise HTTPException(status_code=501, detail="Not implemented") + # --- File content reads ----------------------------------------------- - async def view(self, resource: status_models.Resource, user: User, path: str, size: int, offset: int) -> models.GetViewFileResponse: - raise HTTPException(status_code=501, detail="Not implemented") + async def head( + self, + resource: status_models.Resource, + user: User, + path: str, + file_bytes: int, + lines: int, + skip_trailing: bool, + ) -> models.GetFileHeadResponse: + params: dict = {"path": path} + content_type = models.ContentUnit.lines + if file_bytes is not None: + params["bytes"] = file_bytes + content_type = models.ContentUnit.bytes + if lines is not None: + params["lines"] = lines + text = await _fs_call("GET", "/filesystem/head", params=params) + return models.GetFileHeadResponse(output=_content_payload(text, content_type=content_type)) + + async def tail( + self, + resource: status_models.Resource, + user: User, + path: str, + file_bytes: int | None, + lines: int | None, + skip_heading: bool, + ) -> models.GetFileTailResponse: + params: dict = {"path": path} + content_type = models.ContentUnit.lines + if file_bytes is not None: + params["bytes"] = file_bytes + content_type = models.ContentUnit.bytes + if lines is not None: + params["lines"] = lines + text = await _fs_call("GET", "/filesystem/tail", params=params) + return models.GetFileTailResponse(output=_content_payload(text, content_type=content_type)) + + async def view( + self, + resource: status_models.Resource, + user: User, + path: str, + size: int, + offset: int, + ) -> models.GetViewFileResponse: + text = await _fs_call( + "GET", "/filesystem/view", + params={"path": path, "size": size, "offset": offset}, + ) + return models.GetViewFileResponse( + output=_content_payload(text, content_type=models.ContentUnit.bytes, offset=offset), + ) + + # --- Metadata --------------------------------------------------------- async def checksum(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileChecksumResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call("GET", "/filesystem/checksum", params={"path": path}) + return models.GetFileChecksumResponse(output=models.FileChecksum.model_validate(result)) async def file(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileTypeResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call("GET", "/filesystem/file", params={"path": path}) + # `file` returns a bare string from the dispatcher. + if isinstance(result, dict): + result = result.get("output") or result.get("type") or "" + return models.GetFileTypeResponse(output=str(result) if result is not None else None) async def stat(self, resource: status_models.Resource, user: User, path: str, dereference: bool) -> models.GetFileStatResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "GET", "/filesystem/stat", + params={"path": path, "dereference": str(dereference).lower()}, + ) + return models.GetFileStatResponse(output=models.FileStat.model_validate(result)) + + # --- File management -------------------------------------------------- async def rm(self, resource: status_models.Resource, user: User, path: str) -> models.RemoveResponse: - raise HTTPException(status_code=501, detail="Not implemented") + await _fs_call("DELETE", "/filesystem/rm", params={"path": path}) + return models.RemoveResponse(output=f"Removed {path}") async def mkdir(self, resource: status_models.Resource, user: User, request_model: models.PostMakeDirRequest) -> models.PostMkdirResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "POST", "/filesystem/mkdir", + json_body={"path": request_model.path, "parent": request_model.parent}, + ) + return models.PostMkdirResponse(output=_file_model(result)) + + async def symlink( + self, + resource: status_models.Resource, + user: User, + request_model: models.PostFileSymlinkRequest, + ) -> models.PostFileSymlinkResponse: + result = await _fs_call( + "POST", "/filesystem/symlink", + json_body={"path": request_model.path, "link_path": request_model.link_path}, + ) + return models.PostFileSymlinkResponse(output=_file_model(result)) - async def symlink(self, resource: status_models.Resource, user: User, request_model: models.PostFileSymlinkRequest) -> models.PostFileSymlinkResponse: - raise HTTPException(status_code=501, detail="Not implemented") + # --- Transfer --------------------------------------------------------- async def download(self, resource: status_models.Resource, user: User, path: str) -> models.GetFileDownloadResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call("GET", "/filesystem/download", params={"path": path}) + # fs-facade returns {"content": , "size": }. + content = result.get("content") if isinstance(result, dict) else result + return models.GetFileDownloadResponse(output=content) async def upload(self, resource: status_models.Resource, user: User, path: str, content: str) -> models.PutFileUploadResponse: - raise HTTPException(status_code=501, detail="Not implemented") + try: + raw = base64.b64decode(content) + except (ValueError, TypeError) as exc: + raise HTTPException(status_code=400, detail=f"Invalid base64 upload content: {exc}") from exc + result = await _fs_call( + "POST", "/filesystem/upload", + params={"path": path}, + files={"file": (path.rsplit("/", 1)[-1] or "upload", raw, "application/octet-stream")}, + ) + return models.PutFileUploadResponse( + output=result if isinstance(result, str) else "File uploaded successfully", + ) + + # --- Archive ---------------------------------------------------------- async def compress(self, resource: status_models.Resource, user: User, request_model: models.PostCompressRequest) -> models.PostCompressResponse: - raise HTTPException(status_code=501, detail="Not implemented") + # NOTE: fs-facade does not currently honour `match_pattern`; it is + # silently dropped here. + body = { + "path": request_model.path, + "target_path": request_model.target_path, + "compression": request_model.compression.value if request_model.compression else "gzip", + "dereference": request_model.dereference, + } + result = await _fs_call("POST", "/filesystem/compress", json_body=body) + return models.PostCompressResponse(output=_file_model(result)) async def extract(self, resource: status_models.Resource, user: User, request_model: models.PostExtractRequest) -> models.PostExtractResponse: - raise HTTPException(status_code=501, detail="Not implemented") + body = { + "path": request_model.path, + "target_path": request_model.target_path, + "compression": request_model.compression.value if request_model.compression else "gzip", + } + result = await _fs_call("POST", "/filesystem/extract", json_body=body) + return models.PostExtractResponse(output=_file_model(result)) async def mv(self, resource: status_models.Resource, user: User, request_model: models.PostMoveRequest) -> models.PostMoveResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "POST", "/filesystem/mv", + json_body={"path": request_model.path, "target_path": request_model.target_path}, + ) + return models.PostMoveResponse(output=_file_model(result)) async def cp(self, resource: status_models.Resource, user: User, request_model: models.PostCopyRequest) -> models.PostCopyResponse: - raise HTTPException(status_code=501, detail="Not implemented") + result = await _fs_call( + "POST", "/filesystem/cp", + json_body={ + "path": request_model.path, + "target_path": request_model.target_path, + "dereference": request_model.dereference, + }, + ) + return models.PostCopyResponse(output=_file_model(result)) diff --git a/app/s3df/status/__init__.py b/app/s3df/status/__init__.py deleted file mode 100644 index 9687cc89..00000000 --- a/app/s3df/status/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -S3DF status adapter package. - -Splits the adapter into focused modules: - - * ``config`` — health-check model, resource registry, settings - * ``health_checker`` — Prometheus, InfluxDB, HTTP query + evaluation - * ``store`` — in-memory current-status / event / incident store - * ``poller`` — background polling loop - -The public ``S3DFStatusAdapter`` lives in ``app.s3df.status_adapter``. -""" - -from .config import ( - REGISTRY, - Backend, - Condition, - HealthCheck, - MonitoredResource, - StatusSettings, - build_registry, -) -from .health_checker import HealthChecker, HealthResult, aggregate_results, evaluate -from .poller import StatusPoller -from .store import StatusStore - -__all__ = [ - "REGISTRY", - "Backend", - "Condition", - "HealthCheck", - "MonitoredResource", - "StatusSettings", - "build_registry", - "HealthChecker", - "HealthResult", - "aggregate_results", - "evaluate", - "StatusPoller", - "StatusStore", -] diff --git a/app/s3df/status/config.py b/app/s3df/status/config.py deleted file mode 100644 index 666cb868..00000000 --- a/app/s3df/status/config.py +++ /dev/null @@ -1,490 +0,0 @@ -""" -Configuration for the S3DF status adapter. - -IRI owns the S3DF status runtime: it periodically runs configured health checks, -evaluates status-pusher-style conditions, and caches the latest result for each -resource. The external S3DF status repositories are reference material only; this -module does not fetch dashboard log files. - -Required/optional env vars: - S3DF_PROMETHEUS_URL Prometheus base URL (default https://prometheus.slac.stanford.edu) - S3DF_INFLUXDB_URL InfluxDB base URL (default https://influxdb.slac.stanford.edu) - S3DF_INFLUXDB_DB InfluxDB database (default telegraf) - S3DF_STATUS_CHECKS_FILE JSON file with resource health checks - S3DF_STATUS_CHECKS_JSON JSON mapping resource ids to additional health checks - S3DF_STATUS_REQUIRE_FULL_COVERAGE Require every resource to have checks (default true) - S3DF_STATUS_POLL_INTERVAL Seconds between polls (default 60) - S3DF_STATUS_HTTP_TIMEOUT Per-query timeout sec (default 15) - S3DF_SITE_ID Site id for resources (default s3df) - S3DF_STATUS_TLS_VERIFY true | false | (default false) -""" - -import datetime -import json -import operator -import os -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any, Callable - -from app.routers.status.models import Resource, ResourceType, Status - - -def utc_now() -> datetime.datetime: - return datetime.datetime.now(datetime.timezone.utc) - - -_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) - - -class Backend(str, Enum): - """Backend/source a health check queries.""" - - prometheus = "prometheus" - influxdb = "influxdb" - http = "http" - - -_COMPARATORS: dict[str, Callable[[float, float], bool]] = { - "eq": operator.eq, - "ne": operator.ne, - "lt": operator.lt, - "lte": operator.le, - "gt": operator.gt, - "gte": operator.ge, -} - - -@dataclass(frozen=True) -class Condition: - """A comparison of an observed value against a threshold.""" - - comparator: str - value: float - - def __post_init__(self) -> None: - if self.comparator not in _COMPARATORS: - raise ValueError(f"Unknown comparator: {self.comparator}") - - def met(self, observed: float) -> bool: - return bool(_COMPARATORS[self.comparator](observed, self.value)) - - -@dataclass(frozen=True) -class HealthCheck: - """How to determine a resource's status from an IRI-owned source.""" - - backend: Backend - name: str | None = None - query: str | None = None - up_when: Condition = field(default_factory=lambda: Condition("eq", 1.0)) - db_name: str | None = None - degraded_when: Condition | None = None - url: str | None = None - method: str = "GET" - headers: dict[str, str] = field(default_factory=dict) - follow_redirects: bool = True - - -@dataclass(frozen=True) -class MonitoredResource: - """Pairs a static Resource template with the health checks driving it.""" - - resource: Resource - health_checks: tuple[HealthCheck, ...] = () - - -def _template( - *, - id: str, - name: str, - description: str, - group: str, - resource_type: ResourceType, - capability_ids: tuple[str, ...] = (), -) -> Resource: - """Build a static Resource template. The store overwrites dynamic fields.""" - return Resource( - id=id, - name=name, - description=description, - last_modified=_EPOCH, - site_id="", - group=group, - resource_type=resource_type, - current_status=Status.unknown, - capability_ids=list(capability_ids), - ) - - -RESOURCE_TEMPLATES: tuple[Resource, ...] = ( - _template( - id="s3df-ssh-bastions", - name="SSH Bastions", - description="S3DF SSH bastion hosts for command-line access.", - group="access", - resource_type=ResourceType.service, - ), - _template( - id="s3df-interactive-nodes", - name="Interactive Nodes", - description="S3DF interactive login and analysis nodes.", - group="compute", - resource_type=ResourceType.compute, - ), - _template( - id="s3df-docs", - name="S3DF Docs", - description="S3DF user documentation site.", - group="documentation", - resource_type=ResourceType.website, - ), - _template( - id="s3df-batch-servers", - name="Batch Servers", - description="S3DF batch submission and scheduling servers.", - group="compute", - resource_type=ResourceType.compute, - ), - _template( - id="s3df-slurm", - name="Slurm", - description="S3DF Slurm workload management service.", - group="compute", - resource_type=ResourceType.service, - ), - _template( - id="s3df-monitoring", - name="Monitoring", - description="S3DF monitoring and observability services.", - group="operations", - resource_type=ResourceType.service, - ), - _template( - id="s3df-coact", - name="Coact", - description="S3DF Coact allocation and account service.", - group="accounts", - resource_type=ResourceType.service, - ), - _template( - id="s3df-ondemand", - name="OnDemand", - description="S3DF Open OnDemand web service.", - group="access", - resource_type=ResourceType.website, - ), - _template( - id="s3df-kubernetes", - name="Kubernetes", - description="S3DF Kubernetes platform.", - group="platform", - resource_type=ResourceType.system, - ), - _template( - id="s3df-storage", - name="Storage", - description="S3DF storage services.", - group="storage", - resource_type=ResourceType.storage, - ), - _template( - id="s3df-dtns", - name="DTNs", - description="S3DF data transfer nodes.", - group="data-transfer", - resource_type=ResourceType.network, - ), -) - -RESOURCE_IDS = {resource.id for resource in RESOURCE_TEMPLATES} - -BUILTIN_CHECKS: dict[str, tuple[HealthCheck, ...]] = { - "s3df-ssh-bastions": ( - HealthCheck( - backend=Backend.prometheus, - name="ssh-bastion-port-state", - query="avg( avg_over_time(nmap_port_state{service=`ssh`,group=`s3df`}[5m]) )", - up_when=Condition("eq", 1.0), - ), - ), - "s3df-slurm": ( - HealthCheck( - backend=Backend.influxdb, - name="slurmctld-process", - db_name="telegraf", - query=( - 'SELECT mean("status_code") FROM "monit_process" ' - "WHERE \"service\" = 'slurmctld' AND time > now()-5m" - ), - up_when=Condition("eq", 1.0), - ), - HealthCheck( - backend=Backend.influxdb, - name="slurmdbd-process", - db_name="telegraf", - query=( - 'SELECT mean("status_code") FROM "monit_process" ' - "WHERE \"service\" = 'slurmdbd' AND time > now()-5m" - ), - up_when=Condition("eq", 1.0), - ), - ), -} - - -def build_registry( - settings: "StatusSettings | None" = None, - *, - require_full_coverage: bool | None = None, -) -> list[MonitoredResource]: - """Build monitored resources with built-in plus configured checks.""" - configured = settings.resource_checks if settings is not None else {} - monitored = [ - MonitoredResource( - resource=resource, - health_checks=BUILTIN_CHECKS.get(resource.id, ()) + configured.get(resource.id, ()), - ) - for resource in RESOURCE_TEMPLATES - ] - _validate_unique_check_names(monitored) - - strict = require_full_coverage - if strict is None: - strict = settings.require_full_coverage if settings is not None else False - if strict: - _validate_full_coverage(monitored) - - return monitored - - -def missing_check_resources(monitored: list[MonitoredResource]) -> list[MonitoredResource]: - """Return canonical resources that have no configured health checks.""" - return [resource for resource in monitored if not resource.health_checks] - - -def _validate_full_coverage(monitored: list[MonitoredResource]) -> None: - missing = missing_check_resources(monitored) - if not missing: - return - labels = ", ".join(f"{m.resource.id} ({m.resource.name})" for m in missing) - raise ValueError( - "S3DF status resources missing health checks: " - f"{labels}. Configure checks with S3DF_STATUS_CHECKS_FILE or " - "S3DF_STATUS_CHECKS_JSON, or set S3DF_STATUS_REQUIRE_FULL_COVERAGE=false " - "only for development/test use." - ) - - -def _validate_unique_check_names(monitored: list[MonitoredResource]) -> None: - for resource in monitored: - seen: set[str] = set() - for check in resource.health_checks: - if check.name is None: - continue - if check.name in seen: - raise ValueError( - f"Duplicate S3DF status check name for {resource.resource.id}: {check.name}" - ) - seen.add(check.name) - - -# Static default registry for tests/importers that do not need env-driven checks. -REGISTRY: list[MonitoredResource] = build_registry() - - -class StatusSettings: - """Environment-driven settings for the S3DF status adapter.""" - - def __init__(self) -> None: - self.prometheus_url = os.getenv("S3DF_PROMETHEUS_URL", "https://prometheus.slac.stanford.edu") - self.influxdb_url = os.getenv("S3DF_INFLUXDB_URL", "https://influxdb.slac.stanford.edu") - self.influxdb_db = os.getenv("S3DF_INFLUXDB_DB", "telegraf") - self.require_full_coverage = self._parse_bool( - os.getenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", "true"), - "S3DF_STATUS_REQUIRE_FULL_COVERAGE", - ) - self.resource_checks = self._load_resource_checks() - self.poll_interval = int(os.getenv("S3DF_STATUS_POLL_INTERVAL", "60")) - self.http_timeout = float(os.getenv("S3DF_STATUS_HTTP_TIMEOUT", "15")) - # NOTE: the /facility adapter currently mints a random site uuid per - # process, so this id is not a guaranteed cross-reference yet. Set - # S3DF_SITE_ID once a stable site identifier is established. - self.site_id = os.getenv("S3DF_SITE_ID", "s3df") - self.tls_verify = self._parse_verify(os.getenv("S3DF_STATUS_TLS_VERIFY", "false")) - - @staticmethod - def _parse_bool(raw: str, name: str) -> bool: - low = raw.strip().lower() - if low in ("true", "1", "yes", "on"): - return True - if low in ("false", "0", "no", "off", ""): - return False - raise ValueError(f"{name} must be a boolean value") - - @staticmethod - def _parse_verify(raw: str) -> bool | str: - low = raw.strip().lower() - if low in ("true", "1", "yes", "on"): - return True - if low in ("false", "0", "no", "off", ""): - return False - return raw - - @classmethod - def _load_resource_checks(cls) -> dict[str, tuple[HealthCheck, ...]]: - checks_file = os.getenv("S3DF_STATUS_CHECKS_FILE") - file_checks = cls._parse_resource_checks_file(checks_file) - json_checks = cls._parse_resource_checks( - os.getenv("S3DF_STATUS_CHECKS_JSON", "{}"), - "S3DF_STATUS_CHECKS_JSON", - ) - return cls._merge_resource_checks(file_checks, json_checks) - - @classmethod - def _parse_resource_checks_file(cls, checks_file: str | None) -> dict[str, tuple[HealthCheck, ...]]: - if not checks_file: - return {} - path = Path(checks_file) - try: - raw = path.read_text() - except OSError as exc: - raise ValueError(f"S3DF_STATUS_CHECKS_FILE could not be read: {checks_file}") from exc - return cls._parse_resource_checks(raw, f"S3DF_STATUS_CHECKS_FILE ({checks_file})") - - @staticmethod - def _merge_resource_checks( - *sources: dict[str, tuple[HealthCheck, ...]], - ) -> dict[str, tuple[HealthCheck, ...]]: - merged: dict[str, tuple[HealthCheck, ...]] = {} - for source in sources: - for resource_id, checks in source.items(): - merged[resource_id] = merged.get(resource_id, ()) + checks - return merged - - @classmethod - def _parse_resource_checks(cls, raw: str, source: str) -> dict[str, tuple[HealthCheck, ...]]: - raw = raw.strip() - if not raw: - return {} - data = cls._loads_json_object(raw, source) - if not isinstance(data, dict): - raise ValueError(f"{source} must be an object keyed by resource id") - - parsed: dict[str, tuple[HealthCheck, ...]] = {} - for resource_id, checks in data.items(): - if resource_id not in RESOURCE_IDS: - raise ValueError(f"Unknown S3DF status resource id in {source}: {resource_id}") - if not isinstance(checks, list): - raise ValueError(f"Checks for {resource_id} must be a list") - parsed[resource_id] = tuple( - cls._parse_check(resource_id, idx, check) for idx, check in enumerate(checks) - ) - return parsed - - @staticmethod - def _loads_json_object(raw: str, source: str) -> Any: - def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - data: dict[str, Any] = {} - for key, value in pairs: - if key in data: - raise ValueError(f"{source} contains duplicate key: {key}") - data[key] = value - return data - - try: - return json.loads(raw, object_pairs_hook=reject_duplicate_keys) - except json.JSONDecodeError as exc: - raise ValueError(f"{source} must be valid JSON") from exc - - @classmethod - def _parse_check(cls, resource_id: str, idx: int, raw: Any) -> HealthCheck: - if not isinstance(raw, dict): - raise ValueError(f"Check {idx} for {resource_id} must be an object") - try: - backend = Backend(raw["backend"]) - except KeyError as exc: - raise ValueError(f"Check {idx} for {resource_id} is missing backend") from exc - except ValueError as exc: - raise ValueError( - f"Check {idx} for {resource_id} has unsupported backend: {raw.get('backend')}" - ) from exc - - up_when = cls._parse_condition( - raw.get("up_when"), - default=Condition("eq", 200.0 if backend == Backend.http else 1.0), - ) - degraded_when = cls._parse_condition(raw.get("degraded_when"), default=None) - name = cls._optional_str( - raw.get("name"), - f"Check {idx} for {resource_id} has invalid name", - ) - query = cls._optional_str( - raw.get("query"), - f"Check {idx} for {resource_id} has invalid query", - ) - db_name = cls._optional_str( - raw.get("db_name"), - f"Check {idx} for {resource_id} has invalid db_name", - ) - url = cls._optional_str( - raw.get("url"), - f"Check {idx} for {resource_id} has invalid url", - ) - method = raw.get("method", "GET") - if not isinstance(method, str) or not method.strip(): - raise ValueError(f"Check {idx} for {resource_id} has invalid method") - follow_redirects = raw.get("follow_redirects", True) - if not isinstance(follow_redirects, bool): - raise ValueError(f"Check {idx} for {resource_id} has invalid follow_redirects") - headers = raw.get("headers", {}) - if headers is None: - headers = {} - if not isinstance(headers, dict) or not all( - isinstance(k, str) and isinstance(v, str) for k, v in headers.items() - ): - raise ValueError(f"Check {idx} for {resource_id} has invalid headers") - - check = HealthCheck( - backend=backend, - name=name, - query=query, - up_when=up_when, - db_name=db_name, - degraded_when=degraded_when, - url=url, - method=method.strip().upper(), - headers=headers, - follow_redirects=follow_redirects, - ) - cls._validate_check(resource_id, idx, check) - return check - - @staticmethod - def _optional_str(value: Any, error: str) -> str | None: - if value is None: - return None - if not isinstance(value, str) or not value.strip(): - raise ValueError(error) - return value - - @staticmethod - def _parse_condition(raw: Any, default: Condition | None) -> Condition | None: - if raw is None: - return default - if not isinstance(raw, dict): - raise ValueError("condition must be an object") - try: - comparator = raw["comparator"] - value = raw["value"] - except KeyError as exc: - raise ValueError("condition requires comparator and value") from exc - return Condition(str(comparator), float(value)) - - @staticmethod - def _validate_check(resource_id: str, idx: int, check: HealthCheck) -> None: - if check.backend in (Backend.prometheus, Backend.influxdb) and not check.query: - raise ValueError(f"Check {idx} for {resource_id} requires query") - if check.backend == Backend.http and not check.url: - raise ValueError(f"Check {idx} for {resource_id} requires url") diff --git a/app/s3df/status/health_checker.py b/app/s3df/status/health_checker.py deleted file mode 100644 index e34e5f00..00000000 --- a/app/s3df/status/health_checker.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -Health checker for the S3DF status adapter. - -Runs status-pusher-like checks from IRI-owned configuration against Prometheus, -InfluxDB, or HTTP endpoints, evaluates conditions, and aggregates check results -into one resource status. -""" - -import asyncio -import datetime -import logging -from dataclasses import dataclass -from typing import Sequence - -import httpx - -from app.routers.status.models import Status - -from .config import Backend, HealthCheck, StatusSettings, utc_now - -logger = logging.getLogger(__name__) - - -@dataclass -class HealthResult: - """Outcome of one resource health evaluation.""" - - status: Status - value: float | None - observed_at: datetime.datetime - error: str | None = None - - -def evaluate(check: HealthCheck, value: float | None) -> Status: - """Map an observed scalar value to a Status using the check's conditions.""" - if value is None: - return Status.unknown - if check.up_when.met(value): - return Status.up - if check.degraded_when is not None and check.degraded_when.met(value): - return Status.degraded - return Status.down - - -def aggregate_results(results: Sequence[HealthResult]) -> HealthResult: - """Aggregate multiple check results into one resource status. - - Unknown checks are ignored when at least one check produced a usable signal. - This avoids reporting an outage just because one redundant probe failed. - """ - observed_at = max((result.observed_at for result in results), default=utc_now()) - if not results: - return HealthResult(Status.unknown, None, observed_at, "no health checks configured") - - known = [result for result in results if result.status != Status.unknown] - if not known: - errors = "; ".join(result.error for result in results if result.error) - return HealthResult(Status.unknown, None, observed_at, errors or "all health checks returned unknown") - - if any(result.status == Status.down for result in known): - status = Status.down - elif any(result.status == Status.degraded for result in known): - status = Status.degraded - else: - status = Status.up - - return HealthResult(status, None, observed_at) - - -class HealthChecker: - """Executes health checks over a shared httpx AsyncClient.""" - - def __init__(self, settings: StatusSettings, client: httpx.AsyncClient) -> None: - self.settings = settings - self.client = client - - async def check(self, checks: Sequence[HealthCheck]) -> HealthResult: - if not checks: - return HealthResult(Status.unknown, None, utc_now(), "no health checks configured") - - results = await asyncio.gather(*(self.check_one(check) for check in checks), return_exceptions=True) - normalized: list[HealthResult] = [] - for result in results: - if isinstance(result, HealthResult): - normalized.append(result) - else: - normalized.append(HealthResult(Status.unknown, None, utc_now(), str(result))) - return aggregate_results(normalized) - - async def check_one(self, check: HealthCheck) -> HealthResult: - now = utc_now() - try: - if check.backend == Backend.prometheus: - if check.query is None: - raise ValueError("query is required for prometheus health checks") - value = await self._prometheus_query(check.query) - elif check.backend == Backend.influxdb: - if check.query is None: - raise ValueError("query is required for influxdb health checks") - db = check.db_name or self.settings.influxdb_db - value = await self._influx_query(db, check.query) - elif check.backend == Backend.http: - value = await self._http_query(check) - else: # pragma: no cover - guarded by enum/config validation - return HealthResult(Status.unknown, None, now, f"unknown backend {check.backend}") - except Exception as exc: # noqa: BLE001 - health polling should surface errors as unknown status - label = f"{check.backend.value}:{check.name}" if check.name else check.backend.value - logger.warning("Health check failed (%s): %s", label, exc) - return HealthResult(Status.unknown, None, now, str(exc)) - return HealthResult(evaluate(check, value), value, now) - - async def _prometheus_query(self, query: str) -> float | None: - url = self.settings.prometheus_url.rstrip("/") + "/api/v1/query" - resp = await self.client.get(url, params={"query": query}, timeout=self.settings.http_timeout) - resp.raise_for_status() - data = resp.json() - if data.get("status") != "success": - raise RuntimeError(f"Prometheus returned status={data.get('status')}") - result = data.get("data", {}).get("result", []) - if not result: - return None - return float(result[0]["value"][1]) - - async def _influx_query(self, db: str, query: str) -> float | None: - url = self.settings.influxdb_url.rstrip("/") + "/query" - resp = await self.client.get(url, params={"db": db, "q": query}, timeout=self.settings.http_timeout) - resp.raise_for_status() - data = resp.json() - results = data.get("results", []) - if not results: - return None - series = results[0].get("series", []) - if not series or not series[0].get("values"): - return None - return float(series[0]["values"][-1][1]) - - async def _http_query(self, check: HealthCheck) -> float: - if check.url is None: - raise ValueError("url is required for http health checks") - response = await self.client.request( - check.method, - check.url, - headers=check.headers, - timeout=self.settings.http_timeout, - follow_redirects=check.follow_redirects, - ) - return float(response.status_code) diff --git a/app/s3df/status/poller.py b/app/s3df/status/poller.py deleted file mode 100644 index 63df3e82..00000000 --- a/app/s3df/status/poller.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Background poller for the S3DF status adapter. - -Periodically runs the health checks for each monitored resource and feeds the -results into the :class:`~app.s3df.status.store.StatusStore`. The ``httpx`` -client is created lazily inside the running event loop (see :meth:`start`). -""" - -import asyncio -import logging - -import httpx - -from .config import MonitoredResource, StatusSettings -from .health_checker import HealthChecker -from .store import StatusStore - -logger = logging.getLogger(__name__) - - -class StatusPoller: - """Periodically runs health checks and feeds results into the store.""" - - def __init__(self, store: StatusStore, settings: StatusSettings, monitored: list[MonitoredResource]): - self.store = store - self.settings = settings - self.monitored = monitored - self._client: httpx.AsyncClient | None = None - self._checker: HealthChecker | None = None - self._task: asyncio.Task | None = None - - async def start(self) -> None: - """Create the HTTP client (inside the running loop), run one initial poll, - then launch the periodic background loop.""" - self._client = httpx.AsyncClient(verify=self.settings.tls_verify) - self._checker = HealthChecker(self.settings, self._client) - await self._poll_once() # bounded by per-query timeouts; never raises - self._task = asyncio.create_task(self._run(), name="s3df-status-poller") - logger.info( - "S3DF status poller started (%d resources, interval=%ss)", - len(self.monitored), - self.settings.poll_interval, - ) - - async def _run(self) -> None: - try: - while True: - await asyncio.sleep(self.settings.poll_interval) - await self._poll_once() - except asyncio.CancelledError: - raise - except Exception: # noqa: BLE001 - keep the server alive if the loop dies - logger.exception("S3DF status poller loop crashed") - - async def _poll_once(self) -> None: - assert self._checker is not None - monitored = self.monitored - results = await asyncio.gather( - *(self._checker.check(m.health_checks) for m in monitored), - return_exceptions=True, - ) - for m, res in zip(monitored, results): - if isinstance(res, Exception): - logger.warning("Health check raised for %s: %s", m.resource.id, res) - continue - self.store.record(m.resource.id, res) - - async def aclose(self) -> None: - """Cancel the loop and close the HTTP client. For tests/lifespan wiring.""" - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - self._task = None - if self._client is not None: - await self._client.aclose() - self._client = None diff --git a/app/s3df/status/store.py b/app/s3df/status/store.py deleted file mode 100644 index 398d5663..00000000 --- a/app/s3df/status/store.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -In-memory store for the S3DF status adapter. - -Holds the current status per resource plus an event log and incident set, and -turns poll results into IRI ``Event`` / ``Incident`` objects on status -transitions. State is per-process and volatile (lost on restart, diverges -across uvicorn workers). -""" - -import datetime -import uuid - -from app.routers.status.models import ( - Event, - Incident, - IncidentType, - Resolution, - Resource, - Status, -) - -from .config import utc_now -from .health_checker import HealthResult - - -class StatusStore: - """Holds current status per resource plus an event log and incident set. - - Single-writer model: only the poller calls ``record()``, and ``record()`` has - no internal ``await`` points, so each mutation is atomic with respect to the - asyncio event loop. Reader methods return freshly built/copied lists, so a - request handler never observes a half-applied update. State is per-process and - volatile. - - The store is given the static ``Resource`` templates and is the authority for - their dynamic fields: it overlays ``site_id``, ``current_status`` and - ``last_modified`` when projecting a live view in :meth:`resources`. - """ - - def __init__(self, site_id: str, resources: list[Resource]): - self.site_id = site_id - self._templates: dict[str, Resource] = {r.id: r for r in resources} - self._created_at = utc_now() - self._status: dict[str, Status] = {} - self._last_modified: dict[str, datetime.datetime] = {} - self._events: list[Event] = [] - self._incidents: dict[str, Incident] = {} - self._open_incident: dict[str, str] = {} # resource_id -> incident_id - - # -- writer ------------------------------------------------------------ - - def record(self, resource_id: str, result: HealthResult) -> None: - """Record a poll result, emitting an Event + Incident change on transition.""" - if resource_id not in self._templates: - return - prev = self._status.get(resource_id) - new = result.status - ts = result.observed_at - - if prev == new: - return # steady state — nothing to record - - baseline = prev is None - self._status[resource_id] = new - self._last_modified[resource_id] = ts - - event = self._make_event(resource_id, new, ts, baseline) - - if new in (Status.down, Status.degraded): - inc_id = self._open_incident.get(resource_id) - if inc_id is None: - incident = self._make_incident(resource_id, new, ts) - self._incidents[incident.id] = incident - self._open_incident[resource_id] = incident.id - inc_id = incident.id - else: - incident = self._incidents[inc_id] - incident.status = new - incident.last_modified = ts - event.incident_id = inc_id - self._incidents[inc_id].event_ids.append(event.id) - elif new == Status.up: - inc_id = self._open_incident.pop(resource_id, None) - if inc_id is not None: - incident = self._incidents[inc_id] - incident.status = Status.up - incident.end = ts - incident.resolution = Resolution.completed - incident.last_modified = ts - event.incident_id = inc_id - incident.event_ids.append(event.id) - # Status.unknown: record the event but never open/close incidents — - # a monitoring-backend failure is not a confirmed resource outage. - - self._events.append(event) - - def _make_event(self, resource_id: str, status: Status, ts: datetime.datetime, baseline: bool) -> Event: - rname = self._templates[resource_id].name - verb = "initial status" if baseline else "status changed to" - return Event( - id=str(uuid.uuid4()), - name=f"{rname}: {status.value}", - description=f"{rname} {verb} {status.value}.", - last_modified=ts, - occurred_at=ts, - status=status, - resource_id=resource_id, - ) - - def _make_incident(self, resource_id: str, status: Status, ts: datetime.datetime) -> Incident: - rname = self._templates[resource_id].name - return Incident( - id=str(uuid.uuid4()), - name=f"{rname} {status.value}", - description=f"Automatically opened: {rname} observed {status.value}.", - last_modified=ts, - status=status, - start=ts, - type=IncidentType.unplanned, - resolution=Resolution.unresolved, - resource_ids=[resource_id], - event_ids=[], - ) - - # -- readers ----------------------------------------------------------- - - def resources(self) -> list[Resource]: - """Project live Resource views by overlaying the runtime-owned fields - (site_id, current_status, last_modified) onto the static templates.""" - out: list[Resource] = [] - for id_, tmpl in self._templates.items(): - out.append( - tmpl.model_copy( - update={ - "site_id": self.site_id, - "current_status": self._status.get(id_, Status.unknown), - "last_modified": self._last_modified.get(id_, self._created_at), - } - ) - ) - return out - - def events(self) -> list[Event]: - return sorted(self._events, key=lambda e: (e.occurred_at, e.id)) - - def incidents(self) -> list[Incident]: - return sorted(self._incidents.values(), key=lambda i: (i.start, i.id)) diff --git a/app/s3df/status_adapter.py b/app/s3df/status_adapter.py index 647e4f54..64bfdb2a 100644 --- a/app/s3df/status_adapter.py +++ b/app/s3df/status_adapter.py @@ -1,48 +1,37 @@ """ SLAC S3DF Status Adapter for the IRI Facility API. -Serves the IRI ``/status`` router (resources, events, incidents) from -IRI-owned periodic health checks and a local in-memory status cache. - -Design (see design-docs/s3df-status-adapter.md): - - * Resources are STATIC config (``app.s3df.status.config``). Each pairs an IRI - ``Resource`` template with one or more health checks. Built-in checks are - merged with optional ``S3DF_STATUS_CHECKS_FILE`` and - ``S3DF_STATUS_CHECKS_JSON`` checks at adapter startup; full coverage is - required by default. - * A background poller (``app.s3df.status.poller``) runs every - ``S3DF_STATUS_POLL_INTERVAL`` seconds, queries each resource's configured - sources, maps the aggregate result to a Status (up/down/degraded/unknown), - and records it in an in-memory store (``app.s3df.status.store``). - * The store detects status TRANSITIONS to emit Events and to open/close - unplanned Incidents. - -This module hosts only the ``FacilityAdapter`` implementation; the registry, -health checker, store and poller live in the ``app.s3df.status`` package. - -Lifecycle note: IRI constructs adapters synchronously at import/router-build time -— before the asyncio event loop exists — so the poller and its ``httpx`` client -are created LAZILY on the first request (``_ensure_started``), where the loop is -running. State is in-memory and per-process: it is lost on restart and diverges -across multiple uvicorn workers. Persistence and FastAPI-lifespan shutdown wiring -are tracked as follow-ups in the design doc. +Serves the IRI ``/status`` router (resources, events, incidents) by +forwarding queries to ``s3df-status-api`` and merging dynamic status +with the static resource metadata kept in :mod:`app.s3df.status_registry`. + +The upstream microservice owns the polling, store, and incident +lifecycle — this adapter is a stateless translation layer between +``ResourceStatus``/``StatusEvent``/``Incident`` records and the IRI +``Resource``/``Event``/``Incident`` models. """ -import asyncio import datetime import logging +import uuid from app.routers.status import facility_adapter as status_adapter from app.routers.status import models as status_models - -from .status.config import MonitoredResource, StatusSettings, build_registry -from .status.poller import StatusPoller -from .status.store import StatusStore +from app.s3df.clients import S3DFStatusApiError, get_s3df_status_api_client +from app.s3df.status_registry import ( + S3DF_RESOURCES, + ResourceMeta, + parse_status, + site_id, +) logger = logging.getLogger(__name__) +def _utc_now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + def _paginate(items: list, offset: int | None, limit: int | None) -> list: if offset and offset > 0: items = items[offset:] @@ -51,49 +40,123 @@ def _paginate(items: list, offset: int | None, limit: int | None) -> list: return items +def _parse_dt(value) -> datetime.datetime | None: + if value is None: + return None + if isinstance(value, datetime.datetime): + dt = value + else: + try: + dt = datetime.datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt + + +def _build_resource(meta: ResourceMeta, status_payload: dict | None) -> status_models.Resource: + """Merge static metadata with a ResourceStatus dict from s3df-status-api.""" + last_modified = _utc_now() + current_status = status_models.Status.unknown + if status_payload is not None: + current_status = parse_status(status_payload.get("status")) or status_models.Status.unknown + last_modified = ( + _parse_dt(status_payload.get("last_changed_at")) + or _parse_dt(status_payload.get("last_poll")) + or last_modified + ) + return status_models.Resource( + id=meta.id, + name=meta.name, + description=meta.description, + site_id=site_id(), + group=meta.group, + resource_type=meta.resource_type, + capability_ids=list(meta.capability_ids), + current_status=current_status, + last_modified=last_modified, + ) + + +def _event_id(payload: dict) -> str: + eid = payload.get("id") + if eid is not None: + return str(eid) + # synthesize a stable id when the upstream omits one + return uuid.uuid5( + uuid.NAMESPACE_URL, + f"s3df-event:{payload.get('resource_id')}:{payload.get('occurred_at')}", + ).hex + + +def _build_event(payload: dict) -> status_models.Event | None: + occurred_at = _parse_dt(payload.get("occurred_at")) + if occurred_at is None: + return None + to_status = parse_status(payload.get("to_status")) or status_models.Status.unknown + from_status = payload.get("from_status") or "unknown" + detail = payload.get("detail") + resource_id = payload.get("resource_id") or "" + eid = _event_id(payload) + return status_models.Event( + id=eid, + name=f"{resource_id}: {from_status} -> {to_status.value}", + description=detail, + last_modified=occurred_at, + occurred_at=occurred_at, + status=to_status, + resource_id=resource_id, + incident_id=None, + ) + + +def _build_incident(payload: dict) -> status_models.Incident | None: + opened_at = _parse_dt(payload.get("opened_at")) + if opened_at is None: + return None + resolved_at = _parse_dt(payload.get("resolved_at")) + summary = payload.get("summary") + resource_id = payload.get("resource_id") or "" + incident_id = str(payload.get("id") or "") + if not incident_id: + return None + is_resolved = resolved_at is not None + incident_status = status_models.Status.up if is_resolved else status_models.Status.down + resolution = ( + status_models.Resolution.completed if is_resolved else status_models.Resolution.unresolved + ) + return status_models.Incident( + id=incident_id, + name=f"{resource_id} incident", + description=summary, + last_modified=resolved_at or opened_at, + status=incident_status, + resource_ids=[resource_id] if resource_id else [], + event_ids=[], + start=opened_at, + end=resolved_at, + type=status_models.IncidentType.unplanned, + resolution=resolution, + ) + + class S3DFStatusAdapter(status_adapter.FacilityAdapter): - """IRI status FacilityAdapter backed by internal S3DF health checks. + """IRI status FacilityAdapter backed by the s3df-status-api microservice.""" - Implements every method called by the IRI status router: - get_resources, get_resource, get_events, get_event, - get_incidents, get_incident. + def __init__(self): + self._client = get_s3df_status_api_client() - The status router is unauthenticated, so these methods take no user. - """ + # -- helpers ----------------------------------------------------------- - def __init__( - self, - settings: StatusSettings | None = None, - monitored: list[MonitoredResource] | None = None, - ): - self._settings = settings or StatusSettings() - self._monitored = monitored if monitored is not None else build_registry(self._settings) - self._store = StatusStore( - site_id=self._settings.site_id, - resources=[m.resource for m in self._monitored], - ) - self._poller = StatusPoller(self._store, self._settings, self._monitored) - self._started = False - self._start_lock = asyncio.Lock() - - async def _ensure_started(self) -> None: - """Lazily start the poller on first use (double-checked under a lock).""" - if self._started: - return - async with self._start_lock: - if self._started: - return - try: - await self._poller.start() - self._started = True - except Exception: # noqa: BLE001 - retry on the next request - logger.exception("Failed to start S3DF status poller; will retry") - raise - - async def aclose(self) -> None: - """Release background resources (HTTP client + poller task).""" - await self._poller.aclose() - self._started = False + async def _all_resources(self) -> list[status_models.Resource]: + try: + statuses = await self._client.list_resource_statuses() + except S3DFStatusApiError: + logger.exception("s3df-status-api list_resource_statuses failed; returning unknown statuses") + statuses = [] + by_id = {s.get("resource_id"): s for s in statuses if s.get("resource_id")} + return [_build_resource(meta, by_id.get(meta.id)) for meta in S3DF_RESOURCES.values()] # -- resources --------------------------------------------------------- @@ -110,9 +173,9 @@ async def get_resources( capability=None, site_id: str | None = None, ) -> list[status_models.Resource]: - await self._ensure_started() + resources = await self._all_resources() items = status_models.Resource.find( - self._store.resources(), + resources, name=name, description=description, modified_since=modified_since, @@ -124,12 +187,42 @@ async def get_resources( ) return _paginate(items, offset, limit) - async def get_resource(self, id_: str) -> status_models.Resource: - await self._ensure_started() - return status_models.Resource.find_by_id(self._store.resources(), id_, allow_name=True) + async def get_resource(self, id_: str) -> status_models.Resource | None: + meta = S3DF_RESOURCES.get(id_) + if meta is None: + # allow lookup-by-name as get_resources does (router supports it) + for candidate in S3DF_RESOURCES.values(): + if candidate.name == id_: + meta = candidate + break + if meta is None: + return None + try: + payload = await self._client.get_resource_status(meta.id) + except S3DFStatusApiError: + logger.exception(f"s3df-status-api get_resource_status({meta.id}) failed") + payload = None + return _build_resource(meta, payload) # -- events ------------------------------------------------------------ + async def _all_events( + self, + resource_id: str | None = None, + since: datetime.datetime | None = None, + ) -> list[status_models.Event]: + try: + payloads = await self._client.list_events(resource_id=resource_id, since=since) + except S3DFStatusApiError: + logger.exception("s3df-status-api list_events failed; returning empty list") + return [] + events: list[status_models.Event] = [] + for p in payloads: + ev = _build_event(p) + if ev is not None: + events.append(ev) + return events + async def get_events( self, offset: int, @@ -144,9 +237,9 @@ async def get_events( time_: datetime.datetime | None = None, modified_since: datetime.datetime | None = None, ) -> list[status_models.Event]: - await self._ensure_started() + events = await self._all_events(resource_id=resource_id, since=from_) items = status_models.Event.find( - self._store.events(), + events, incident_id=incident_id, resource_id=resource_id, name=name, @@ -159,12 +252,29 @@ async def get_events( ) return _paginate(items, offset, limit) - async def get_event(self, id_: str) -> status_models.Event: - await self._ensure_started() - return status_models.Event.find_by_id(self._store.events(), id_) + async def get_event(self, id_: str) -> status_models.Event | None: + events = await self._all_events() + return status_models.Event.find_by_id(events, id_) # -- incidents --------------------------------------------------------- + async def _all_incidents( + self, + resource_id: str | None = None, + state: str | None = None, + ) -> list[status_models.Incident]: + try: + payloads = await self._client.list_incidents(resource_id=resource_id, state=state) + except S3DFStatusApiError: + logger.exception("s3df-status-api list_incidents failed; returning empty list") + return [] + incidents: list[status_models.Incident] = [] + for p in payloads: + inc = _build_incident(p) + if inc is not None: + incidents.append(inc) + return incidents + async def get_incidents( self, offset: int, @@ -180,9 +290,9 @@ async def get_incidents( resource_id: str | None = None, resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: - await self._ensure_started() + incidents = await self._all_incidents(resource_id=resource_id) items = status_models.Incident.find( - self._store.incidents(), + incidents, name=name, description=description, status=status, @@ -196,6 +306,6 @@ async def get_incidents( ) return _paginate(items, offset, limit) - async def get_incident(self, id_: str) -> status_models.Incident: - await self._ensure_started() - return status_models.Incident.find_by_id(self._store.incidents(), id_) + async def get_incident(self, id_: str) -> status_models.Incident | None: + incidents = await self._all_incidents() + return status_models.Incident.find_by_id(incidents, id_) diff --git a/app/s3df/status_registry.py b/app/s3df/status_registry.py new file mode 100644 index 00000000..8c4551d1 --- /dev/null +++ b/app/s3df/status_registry.py @@ -0,0 +1,132 @@ +""" +Static metadata for the S3DF resources monitored by ``s3df-status-api``. + +The upstream microservice only exposes ``ResourceStatus`` records (id + +current status). The IRI ``/status/resources`` endpoint, in contrast, +must surface the full ``Resource`` model (id, name, description, group, +resource_type, site_id). This module ships a static map so the IRI +adapter can merge dynamic status with descriptive metadata. + +Mirror of ``s3df-status-api/resources.yaml``. +""" + +from dataclasses import dataclass, field + +from app.routers.status import models as status_models +from app.s3df.config import settings + + +@dataclass(frozen=True) +class ResourceMeta: + """Static metadata for an S3DF status resource.""" + id: str + name: str + description: str + group: str + resource_type: status_models.ResourceType + capability_ids: list[str] = field(default_factory=list) + + +# 11 resources, ids/names/groups from s3df-status-api/resources.yaml. +S3DF_RESOURCES: dict[str, ResourceMeta] = { + meta.id: meta + for meta in ( + ResourceMeta( + id="s3df-ssh-bastions", + name="SSH Bastions", + description="S3DF SSH bastion hosts for command-line access.", + group="access", + resource_type=status_models.ResourceType.service, + ), + ResourceMeta( + id="s3df-interactive-nodes", + name="Interactive Nodes", + description="S3DF interactive login and analysis nodes.", + group="compute", + resource_type=status_models.ResourceType.compute, + ), + ResourceMeta( + id="s3df-docs", + name="S3DF Docs", + description="S3DF user documentation site.", + group="documentation", + resource_type=status_models.ResourceType.website, + ), + ResourceMeta( + id="s3df-batch-servers", + name="Batch Servers", + description="S3DF batch submission and scheduling servers.", + group="compute", + resource_type=status_models.ResourceType.compute, + ), + ResourceMeta( + id="s3df-slurm", + name="Slurm", + description="S3DF Slurm workload management service.", + group="compute", + resource_type=status_models.ResourceType.service, + ), + ResourceMeta( + id="s3df-monitoring", + name="Monitoring", + description="S3DF monitoring and observability services.", + group="operations", + resource_type=status_models.ResourceType.service, + ), + ResourceMeta( + id="s3df-coact", + name="Coact", + description="S3DF Coact allocation and account service.", + group="accounts", + resource_type=status_models.ResourceType.service, + ), + ResourceMeta( + id="s3df-ondemand", + name="OnDemand", + description="S3DF Open OnDemand web service.", + group="access", + resource_type=status_models.ResourceType.website, + ), + ResourceMeta( + id="s3df-kubernetes", + name="Kubernetes", + description="S3DF Kubernetes platform.", + group="platform", + resource_type=status_models.ResourceType.system, + ), + ResourceMeta( + id="s3df-storage", + name="Storage", + description="S3DF storage services.", + group="storage", + resource_type=status_models.ResourceType.storage, + ), + ResourceMeta( + id="s3df-dtns", + name="DTNs", + description="S3DF data transfer nodes.", + group="data-transfer", + resource_type=status_models.ResourceType.network, + ), + ) +} + + +def site_id() -> str: + """The site id used for every S3DF status resource.""" + return settings.facility_name or "s3df" + + +_STATUS_MAP = { + "up": status_models.Status.up, + "down": status_models.Status.down, + "degraded": status_models.Status.degraded, + "unknown": status_models.Status.unknown, +} + + +def parse_status(value: str | None) -> status_models.Status | None: + """Map an upstream status string to the IRI Status enum.""" + if value is None: + return None + return _STATUS_MAP.get(value, status_models.Status.unknown) diff --git a/app/s3df/tests/test_filesystem_adapter.py b/app/s3df/tests/test_filesystem_adapter.py new file mode 100644 index 00000000..f16cdbe0 --- /dev/null +++ b/app/s3df/tests/test_filesystem_adapter.py @@ -0,0 +1,170 @@ +"""Tests for the S3DF filesystem adapter (fs-facade-service backed).""" + +import base64 +import datetime +import json + +import httpx +import pytest + +from app.routers.filesystem import models +from app.routers.status import models as status_models +from app.s3df import filesystem_adapter as filesystem_adapter_module +from app.s3df.clients import fs_facade +from app.s3df.filesystem_adapter import S3DFFilesystemAdapter +from app.types.user import User + + +def _resource() -> status_models.Resource: + return status_models.Resource( + id="s3df-storage", + name="Storage", + description="storage", + site_id="s3df", + group="storage", + resource_type=status_models.ResourceType.storage, + current_status=status_models.Status.up, + last_modified=datetime.datetime(2026, 6, 1, tzinfo=datetime.timezone.utc), + ) + + +def _user() -> User: + return User(id="amithm", name="amithm", api_key="test", client_ip="127.0.0.1") + + +def _file_dict(name: str = "f.txt") -> dict: + return { + "name": name, + "type": "file", + "user": "amithm", + "group": "amithm", + "permissions": "rw-r--r--", + "last_modified": "2026-06-01 12:00:00", + "size": "10", + } + + +def _install_handler(monkeypatch, handler): + """Replace the global fs-facade client with one wired to a MockTransport.""" + transport = httpx.MockTransport(handler) + client = fs_facade.FsFacadeClient(base_url="http://fs-facade.test", poll_interval=0, timeout=5) + client._client = httpx.AsyncClient(base_url=client.base_url, transport=transport) + monkeypatch.setattr(filesystem_adapter_module, "get_fs_facade_client", lambda: client) + return client + + +def _task_response(result, *, status: str = "completed", task_id: str = "t-1") -> httpx.Response: + body = { + "output": { + "id": task_id, + "status": status, + "result": result, + "command": None, + } + } + return httpx.Response(200, json=body) + + +@pytest.mark.asyncio +async def test_chmod_returns_file_model(monkeypatch): + captured: dict = {} + + def handler(req: httpx.Request) -> httpx.Response: + if req.method == "PUT" and req.url.path == "/filesystem/chmod": + captured["body"] = json.loads(req.content) + return httpx.Response(200, json="task-chmod") + if req.url.path == "/task/task-chmod": + return _task_response(json.dumps(_file_dict("a.txt"))) + return httpx.Response(404) + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + response = await adapter.chmod( + _resource(), _user(), + models.PutFileChmodRequest(path="/sdf/data/a.txt", mode="755"), + ) + assert isinstance(response, models.PutFileChmodResponse) + assert response.output.name == "a.txt" + assert captured["body"] == {"path": "/sdf/data/a.txt", "mode": "755"} + + +@pytest.mark.asyncio +async def test_ls_returns_list_of_files(monkeypatch): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/filesystem/ls": + return httpx.Response(200, json="task-ls") + if req.url.path == "/task/task-ls": + payload = json.dumps([_file_dict("a.txt"), _file_dict("b.txt")]) + return _task_response(payload) + return httpx.Response(404) + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + response = await adapter.ls( + _resource(), _user(), + path="/sdf/data", show_hidden=False, + numeric_uid=False, recursive=False, dereference=False, + ) + assert isinstance(response, models.GetDirectoryLsResponse) + assert [f.name for f in response.output] == ["a.txt", "b.txt"] + + +@pytest.mark.asyncio +async def test_download_returns_base64_content(monkeypatch): + payload = json.dumps({"content": base64.b64encode(b"hello").decode(), "size": 5}) + + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/filesystem/download": + return httpx.Response(200, json="task-dl") + if req.url.path == "/task/task-dl": + return _task_response(payload) + return httpx.Response(404) + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + response = await adapter.download(_resource(), _user(), path="/sdf/data/a.txt") + assert response.output == base64.b64encode(b"hello").decode() + + +@pytest.mark.asyncio +async def test_rm_returns_remove_response(monkeypatch): + def handler(req: httpx.Request) -> httpx.Response: + if req.method == "DELETE" and req.url.path == "/filesystem/rm": + return httpx.Response(200, json="task-rm") + if req.url.path == "/task/task-rm": + return _task_response(json.dumps({"status": "ok", "path": "/sdf/data/a.txt"})) + return httpx.Response(404) + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + response = await adapter.rm(_resource(), _user(), path="/sdf/data/a.txt") + assert isinstance(response, models.RemoveResponse) + assert "/sdf/data/a.txt" in response.output + + +@pytest.mark.asyncio +async def test_facade_5xx_maps_to_502(monkeypatch): + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + with pytest.raises(Exception) as excinfo: + await adapter.rm(_resource(), _user(), path="/sdf/data/a.txt") + assert getattr(excinfo.value, "status_code", None) == 502 + + +@pytest.mark.asyncio +async def test_failed_task_maps_to_502(monkeypatch): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/filesystem/rm": + return httpx.Response(200, json="task-fail") + if req.url.path == "/task/task-fail": + return _task_response("Error: permission denied", status="failed") + return httpx.Response(404) + + _install_handler(monkeypatch, handler) + adapter = S3DFFilesystemAdapter() + with pytest.raises(Exception) as excinfo: + await adapter.rm(_resource(), _user(), path="/sdf/data/a.txt") + assert getattr(excinfo.value, "status_code", None) == 502 diff --git a/app/s3df/tests/test_status_adapter.py b/app/s3df/tests/test_status_adapter.py index 204a5f9a..e7199026 100644 --- a/app/s3df/tests/test_status_adapter.py +++ b/app/s3df/tests/test_status_adapter.py @@ -1,444 +1,169 @@ +"""Tests for the S3DF status adapter (microservice-backed).""" + import datetime -import json +from unittest.mock import AsyncMock -import httpx import pytest -from app.routers.status.models import ResourceType, Status -from app.s3df.status.config import ( - Backend, - Condition, - HealthCheck, - MonitoredResource, - REGISTRY, - RESOURCE_TEMPLATES, - StatusSettings, - build_registry, - missing_check_resources, -) -from app.s3df.status.health_checker import HealthChecker, HealthResult, aggregate_results -from app.s3df.status.store import StatusStore +from app.routers.status import models as status_models +from app.s3df import status_adapter as status_adapter_module +from app.s3df.clients import S3DFStatusApiError from app.s3df.status_adapter import S3DFStatusAdapter - - -EXPECTED_NAMES = [ - "SSH Bastions", - "Interactive Nodes", - "S3DF Docs", - "Batch Servers", - "Slurm", - "Monitoring", - "Coact", - "OnDemand", - "Kubernetes", - "Storage", - "DTNs", -] - -EXPECTED_IDS = [ - "s3df-ssh-bastions", - "s3df-interactive-nodes", - "s3df-docs", - "s3df-batch-servers", - "s3df-slurm", - "s3df-monitoring", - "s3df-coact", - "s3df-ondemand", - "s3df-kubernetes", - "s3df-storage", - "s3df-dtns", -] - -NOW = datetime.datetime(2026, 6, 1, 13, 0, tzinfo=datetime.timezone.utc) - - -def _clear_status_check_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("S3DF_STATUS_CHECKS_FILE", raising=False) - monkeypatch.delenv("S3DF_STATUS_CHECKS_JSON", raising=False) - monkeypatch.delenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", raising=False) - - -def _full_coverage_config() -> dict[str, list[dict[str, object]]]: - builtin_ids = {"s3df-ssh-bastions", "s3df-slurm"} - return { - resource.id: [ - { - "backend": "http", - "name": f"{resource.id}-status", - "url": f"https://status.example/{resource.id}", - "up_when": {"comparator": "eq", "value": 200}, - } - ] - for resource in RESOURCE_TEMPLATES - if resource.id not in builtin_ids - } - - -def _settings(monkeypatch: pytest.MonkeyPatch) -> StatusSettings: - _clear_status_check_env(monkeypatch) - settings = StatusSettings() - settings.prometheus_url = "https://prometheus.example" - settings.influxdb_url = "https://influx.example" - settings.poll_interval = 3600 - settings.http_timeout = 1 - return settings - - -def test_registry_matches_s3df_status_resources(): - assert [m.resource.name for m in REGISTRY] == EXPECTED_NAMES - assert [m.resource.id for m in REGISTRY] == EXPECTED_IDS - assert {m.resource.id for m in REGISTRY}.isdisjoint({"s3df-ssh-gateway", "s3df-slurmctld", "s3df-slurmdbd"}) - - checks_by_id = {m.resource.id: m.health_checks for m in REGISTRY} - assert [check.backend for check in checks_by_id["s3df-ssh-bastions"]] == [Backend.prometheus] - assert [check.backend for check in checks_by_id["s3df-slurm"]] == [Backend.influxdb, Backend.influxdb] - assert all(not checks for id_, checks in checks_by_id.items() if id_ not in {"s3df-ssh-bastions", "s3df-slurm"}) - - -def test_build_registry_requires_full_coverage_by_default(monkeypatch): - _clear_status_check_env(monkeypatch) - - with pytest.raises(ValueError, match="S3DF status resources missing health checks") as exc_info: - build_registry(StatusSettings()) - - assert "s3df-interactive-nodes" in str(exc_info.value) - assert "S3DF_STATUS_CHECKS_FILE" in str(exc_info.value) - - -def test_build_registry_allows_missing_checks_only_when_explicitly_non_strict(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv("S3DF_STATUS_REQUIRE_FULL_COVERAGE", "false") - - registry = build_registry(StatusSettings()) - - assert [m.resource.id for m in missing_check_resources(registry)] == [ - resource.id - for resource in RESOURCE_TEMPLATES - if resource.id not in {"s3df-ssh-bastions", "s3df-slurm"} +from app.s3df.status_registry import S3DF_RESOURCES + + +def _statuses_payload(overrides: dict[str, str] | None = None) -> list[dict]: + """Build a synthetic ResourceStatus list covering every registry id.""" + overrides = overrides or {} + now = datetime.datetime(2026, 6, 1, tzinfo=datetime.timezone.utc).isoformat() + return [ + { + "resource_id": rid, + "status": overrides.get(rid, "up"), + "last_changed_at": now, + "last_poll": now, + "check_results": [], + } + for rid in S3DF_RESOURCES ] -def test_status_log_runtime_configuration_is_not_available(monkeypatch): - settings = _settings(monkeypatch) - - assert "status_log" not in Backend.__members__ - assert not hasattr(settings, "status_logs_base_url") - assert not hasattr(settings, "status_log_max_age") - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (200, Status.up), - (499, Status.degraded), - (500, Status.down), - (None, Status.unknown), - ], -) -def test_condition_evaluation_with_degraded_status(value, expected): - check = HealthCheck( - backend=Backend.http, - url="https://example/status", - up_when=Condition("eq", 200), - degraded_when=Condition("lt", 500), - ) +def _make_adapter(monkeypatch, *, list_resources=None, get_resource=None, list_events=None, list_incidents=None): + """Construct an adapter wired to a mock client.""" + mock_client = AsyncMock() + mock_client.list_resource_statuses = AsyncMock(return_value=list_resources or _statuses_payload()) + mock_client.get_resource_status = AsyncMock(side_effect=get_resource) + mock_client.list_events = AsyncMock(return_value=list_events or []) + mock_client.list_incidents = AsyncMock(return_value=list_incidents or []) + monkeypatch.setattr(status_adapter_module, "get_s3df_status_api_client", lambda: mock_client) + return S3DFStatusAdapter(), mock_client - from app.s3df.status.health_checker import evaluate - assert evaluate(check, value) == expected - - -@pytest.mark.parametrize( - ("results", "expected"), - [ - ([], Status.unknown), - ([HealthResult(Status.unknown, None, NOW, "backend unavailable")], Status.unknown), - ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.unknown, None, NOW, "timeout")], Status.up), - ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.degraded, 0.5, NOW)], Status.degraded), - ([HealthResult(Status.up, 1.0, NOW), HealthResult(Status.down, 0.0, NOW)], Status.down), - ], -) -def test_aggregate_results(results, expected): - assert aggregate_results(results).status == expected +@pytest.mark.asyncio +async def test_get_resources_merges_registry_and_status(monkeypatch): + adapter, _ = _make_adapter(monkeypatch) + resources = await adapter.get_resources(offset=0, limit=100) + assert len(resources) == len(S3DF_RESOURCES) + by_id = {r.id: r for r in resources} + bastions = by_id["s3df-ssh-bastions"] + assert bastions.name == "SSH Bastions" + assert bastions.group == "access" + assert bastions.resource_type is status_models.ResourceType.service + assert bastions.current_status is status_models.Status.up @pytest.mark.asyncio -async def test_health_checker_queries_prometheus(monkeypatch): - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/api/v1/query" - assert request.url.params["query"] == "up" - return httpx.Response(200, json={"status": "success", "data": {"result": [{"value": [0, "1"]}]}}) +async def test_get_resources_filter_by_group(monkeypatch): + adapter, _ = _make_adapter(monkeypatch) + resources = await adapter.get_resources(offset=0, limit=100, group="compute") + ids = {r.id for r in resources} + assert ids == {"s3df-interactive-nodes", "s3df-batch-servers", "s3df-slurm"} - settings = _settings(monkeypatch) - async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: - result = await HealthChecker(settings, client).check( - (HealthCheck(backend=Backend.prometheus, query="up", up_when=Condition("eq", 1)),) - ) - assert result.status == Status.up +@pytest.mark.asyncio +async def test_get_resources_filter_by_current_status(monkeypatch): + payload = _statuses_payload({"s3df-storage": "down", "s3df-dtns": "degraded"}) + adapter, _ = _make_adapter(monkeypatch, list_resources=payload) + down = await adapter.get_resources(offset=0, limit=100, current_status=status_models.Status.down) + assert [r.id for r in down] == ["s3df-storage"] @pytest.mark.asyncio -async def test_health_checker_queries_influxdb(monkeypatch): - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/query" - assert request.url.params["db"] == "telegraf" - assert request.url.params["q"] == "SELECT mean(status_code)" - return httpx.Response(200, json={"results": [{"series": [{"values": [["2026-06-01T12:00:00Z", 0.5]]}]}]}) +async def test_get_resources_falls_back_when_api_fails(monkeypatch): + async def boom(*_, **__): + raise S3DFStatusApiError("boom") - settings = _settings(monkeypatch) - async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: - result = await HealthChecker(settings, client).check( - ( - HealthCheck( - backend=Backend.influxdb, - query="SELECT mean(status_code)", - up_when=Condition("eq", 1), - degraded_when=Condition("gte", 0.5), - ), - ) - ) - - assert result.status == Status.degraded + adapter = S3DFStatusAdapter.__new__(S3DFStatusAdapter) + adapter._client = AsyncMock() + adapter._client.list_resource_statuses = AsyncMock(side_effect=boom) + resources = await adapter.get_resources(offset=0, limit=100) + assert len(resources) == len(S3DF_RESOURCES) + assert all(r.current_status is status_models.Status.unknown for r in resources) @pytest.mark.asyncio -async def test_health_checker_queries_http_status(monkeypatch): - def handler(request: httpx.Request) -> httpx.Response: - assert str(request.url) == "https://docs.example/status" - assert request.headers["x-check"] == "docs" - return httpx.Response(204) - - settings = _settings(monkeypatch) - async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: - result = await HealthChecker(settings, client).check( - ( - HealthCheck( - backend=Backend.http, - url="https://docs.example/status", - headers={"x-check": "docs"}, - up_when=Condition("eq", 204), - ), - ) - ) - - assert result.status == Status.up +async def test_get_resource_unknown_returns_none(monkeypatch): + adapter, _ = _make_adapter(monkeypatch, get_resource=lambda *_a, **_k: None) + assert await adapter.get_resource("not-a-real-id") is None @pytest.mark.asyncio -async def test_health_checker_maps_backend_failures_to_unknown(monkeypatch): - settings = _settings(monkeypatch) - async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(500))) as client: - result = await HealthChecker(settings, client).check( - (HealthCheck(backend=Backend.prometheus, query="up", up_when=Condition("eq", 1)),) - ) - - assert result.status == Status.unknown - assert result.error is not None +async def test_get_resource_known_id(monkeypatch): + async def get_resource(_id): + return { + "resource_id": "s3df-coact", + "status": "degraded", + "last_changed_at": "2026-06-01T12:00:00Z", + "last_poll": "2026-06-01T12:00:00Z", + "check_results": [], + } + + adapter = S3DFStatusAdapter.__new__(S3DFStatusAdapter) + adapter._client = AsyncMock() + adapter._client.get_resource_status = AsyncMock(side_effect=get_resource) + res = await adapter.get_resource("s3df-coact") + assert res is not None + assert res.id == "s3df-coact" + assert res.current_status is status_models.Status.degraded -def test_configured_checks_are_merged_with_builtin_checks(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv( - "S3DF_STATUS_CHECKS_JSON", - json.dumps( - { - "s3df-docs": [ - { - "backend": "http", - "name": "docs-health", - "url": "https://docs.example/status", - "up_when": {"comparator": "eq", "value": 204}, - } - ], - "s3df-slurm": [ - { - "backend": "prometheus", - "name": "slurm-exporter", - "query": "slurm_up", - "up_when": {"comparator": "gte", "value": 1}, - } - ], - } - ), - ) +@pytest.mark.asyncio +async def test_get_events_returns_empty_when_disabled(monkeypatch): + adapter, _ = _make_adapter(monkeypatch, list_events=[]) + assert await adapter.get_events(offset=0, limit=100) == [] - registry = build_registry(StatusSettings(), require_full_coverage=False) - checks_by_id = {m.resource.id: m.health_checks for m in registry} - assert checks_by_id["s3df-docs"][0].backend == Backend.http - assert checks_by_id["s3df-docs"][0].up_when == Condition("eq", 204) - assert [check.name for check in checks_by_id["s3df-slurm"]] == [ - "slurmctld-process", - "slurmdbd-process", - "slurm-exporter", +@pytest.mark.asyncio +async def test_get_events_maps_payloads(monkeypatch): + payloads = [ + { + "id": 1, + "resource_id": "s3df-coact", + "from_status": "up", + "to_status": "down", + "occurred_at": "2026-06-01T12:00:00Z", + "detail": "probe failed", + } ] - - -def test_check_file_can_provide_full_coverage(monkeypatch, tmp_path): - _clear_status_check_env(monkeypatch) - check_file = tmp_path / "s3df-status-checks.json" - check_file.write_text(json.dumps(_full_coverage_config())) - monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", str(check_file)) - - registry = build_registry(StatusSettings()) - checks_by_id = {m.resource.id: m.health_checks for m in registry} - - assert not missing_check_resources(registry) - assert checks_by_id["s3df-docs"][0].backend == Backend.http - assert checks_by_id["s3df-docs"][0].url == "https://status.example/s3df-docs" - - -def test_check_file_and_inline_json_are_merged(monkeypatch, tmp_path): - _clear_status_check_env(monkeypatch) - check_file = tmp_path / "s3df-status-checks.json" - check_file.write_text( - json.dumps( - { - "s3df-docs": [ - { - "backend": "http", - "name": "docs-file", - "url": "https://docs.example/from-file", - } - ] - } - ) - ) - monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", str(check_file)) - monkeypatch.setenv( - "S3DF_STATUS_CHECKS_JSON", - json.dumps( - { - "s3df-docs": [ - { - "backend": "http", - "name": "docs-inline", - "url": "https://docs.example/from-inline", - } - ] - } - ), - ) - - registry = build_registry(StatusSettings(), require_full_coverage=False) - docs_checks = {m.resource.id: m.health_checks for m in registry}["s3df-docs"] - - assert [check.name for check in docs_checks] == ["docs-file", "docs-inline"] - - -def test_invalid_configured_check_resource_id_is_explicit(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv("S3DF_STATUS_CHECKS_JSON", json.dumps({"s3df-missing": []})) - - with pytest.raises(ValueError, match="Unknown S3DF status resource id"): - StatusSettings() - - -def test_duplicate_json_keys_are_explicit(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv("S3DF_STATUS_CHECKS_JSON", '{"s3df-docs": [], "s3df-docs": []}') - - with pytest.raises(ValueError, match="duplicate key: s3df-docs"): - StatusSettings() - - -def test_duplicate_check_names_are_explicit(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv( - "S3DF_STATUS_CHECKS_JSON", - json.dumps( - { - "s3df-slurm": [ - { - "backend": "prometheus", - "name": "slurmctld-process", - "query": "slurm_up", - } - ] - } - ), - ) - - with pytest.raises(ValueError, match="Duplicate S3DF status check name"): - build_registry(StatusSettings(), require_full_coverage=False) - - -def test_missing_check_file_is_explicit(monkeypatch): - _clear_status_check_env(monkeypatch) - monkeypatch.setenv("S3DF_STATUS_CHECKS_FILE", "/does/not/exist.json") - - with pytest.raises(ValueError, match="S3DF_STATUS_CHECKS_FILE could not be read"): - StatusSettings() + adapter, _ = _make_adapter(monkeypatch, list_events=payloads) + events = await adapter.get_events(offset=0, limit=100) + assert len(events) == 1 + e = events[0] + assert e.id == "1" + assert e.status is status_models.Status.down + assert e.resource_id == "s3df-coact" + assert e.description == "probe failed" @pytest.mark.asyncio -async def test_adapter_returns_cached_resources_with_current_status_and_filters(monkeypatch): - observed_at = datetime.datetime(2026, 6, 1, 12, 0, tzinfo=datetime.timezone.utc) - statuses = { - "s3df-kubernetes": Status.degraded, - "s3df-storage": Status.down, - } - - monitored = [ - MonitoredResource( - resource=resource, - health_checks=(HealthCheck(backend=Backend.http, name=resource.id, url=f"https://status.example/{resource.id}"),), - ) - for resource in RESOURCE_TEMPLATES +async def test_get_incidents_maps_open_and_resolved(monkeypatch): + payloads = [ + { + "id": "inc-1", + "resource_id": "s3df-storage", + "opened_at": "2026-06-01T12:00:00Z", + "resolved_at": None, + "summary": "weka degraded", + }, + { + "id": "inc-2", + "resource_id": "s3df-coact", + "opened_at": "2026-06-01T11:00:00Z", + "resolved_at": "2026-06-01T11:30:00Z", + "summary": "transient", + }, ] + adapter, _ = _make_adapter(monkeypatch, list_incidents=payloads) + incidents = await adapter.get_incidents(offset=0, limit=100) + assert len(incidents) == 2 + by_id = {i.id: i for i in incidents} + assert by_id["inc-1"].resolution is status_models.Resolution.unresolved + assert by_id["inc-1"].status is status_models.Status.down + assert by_id["inc-2"].resolution is status_models.Resolution.completed + assert by_id["inc-2"].status is status_models.Status.up - async def fake_check(self, checks): - return HealthResult(statuses.get(checks[0].name, Status.up), None, observed_at) - - monkeypatch.setattr(HealthChecker, "check", fake_check) - adapter = S3DFStatusAdapter(settings=_settings(monkeypatch), monitored=monitored) - try: - resources = await adapter.get_resources(0, 100) - - assert [r.name for r in resources] == EXPECTED_NAMES - assert [r.id for r in resources] == EXPECTED_IDS - assert {r.current_status for r in resources} == {Status.up, Status.degraded, Status.down} - assert all(r.last_modified == observed_at for r in resources) - assert [r.name for r in await adapter.get_resources(0, 100, current_status=Status.down)] == ["Storage"] - assert [r.name for r in await adapter.get_resources(0, 100, group="access")] == ["SSH Bastions", "OnDemand"] - assert [r.name for r in await adapter.get_resources(0, 100, resource_type=ResourceType.website)] == ["S3DF Docs", "OnDemand"] - assert len(await adapter.get_resources(0, 100, site_id="s3df")) == len(EXPECTED_NAMES) - assert (await adapter.get_resource("Storage")).id == "s3df-storage" - finally: - await adapter.aclose() - - -def test_store_incident_lifecycle_with_cached_check_results(): - resource = REGISTRY[0].resource - store = StatusStore("s3df", [resource]) - ts1 = datetime.datetime(2026, 6, 1, 12, 0, tzinfo=datetime.timezone.utc) - ts2 = datetime.datetime(2026, 6, 1, 12, 5, tzinfo=datetime.timezone.utc) - ts3 = datetime.datetime(2026, 6, 1, 12, 10, tzinfo=datetime.timezone.utc) - ts4 = datetime.datetime(2026, 6, 1, 12, 15, tzinfo=datetime.timezone.utc) - - store.record(resource.id, HealthResult(Status.up, None, ts1)) - assert len(store.events()) == 1 - assert store.incidents() == [] - - store.record(resource.id, HealthResult(Status.down, None, ts2)) - incident = store.incidents()[0] - assert incident.status == Status.down - assert incident.resolution.value == "unresolved" - assert incident.start == ts2 - assert store.events()[-1].incident_id == incident.id - - store.record(resource.id, HealthResult(Status.unknown, None, ts3)) - assert store.incidents()[0].id == incident.id - assert store.incidents()[0].end is None - assert store.events()[-1].status == Status.unknown - assert store.events()[-1].incident_id is None - - store.record(resource.id, HealthResult(Status.up, None, ts4)) - closed = store.incidents()[0] - assert closed.id == incident.id - assert closed.status == Status.up - assert closed.end == ts4 - assert closed.resolution.value == "completed" - assert store.events()[-1].incident_id == incident.id +@pytest.mark.asyncio +async def test_get_incidents_returns_empty_when_disabled(monkeypatch): + adapter, _ = _make_adapter(monkeypatch, list_incidents=[]) + assert await adapter.get_incidents(offset=0, limit=100) == [] diff --git a/app/s3df/tests/test_status_registry.py b/app/s3df/tests/test_status_registry.py new file mode 100644 index 00000000..74612483 --- /dev/null +++ b/app/s3df/tests/test_status_registry.py @@ -0,0 +1,55 @@ +"""Tests for the static S3DF status registry.""" + +from app.routers.status import models as status_models +from app.s3df.status_registry import S3DF_RESOURCES, parse_status, site_id + + +EXPECTED_IDS = [ + "s3df-ssh-bastions", + "s3df-interactive-nodes", + "s3df-docs", + "s3df-batch-servers", + "s3df-slurm", + "s3df-monitoring", + "s3df-coact", + "s3df-ondemand", + "s3df-kubernetes", + "s3df-storage", + "s3df-dtns", +] + + +def test_registry_has_all_expected_resources(): + assert list(S3DF_RESOURCES.keys()) == EXPECTED_IDS + + +def test_registry_resource_types_match_yaml(): + by_id = S3DF_RESOURCES + assert by_id["s3df-ssh-bastions"].resource_type is status_models.ResourceType.service + assert by_id["s3df-interactive-nodes"].resource_type is status_models.ResourceType.compute + assert by_id["s3df-docs"].resource_type is status_models.ResourceType.website + assert by_id["s3df-storage"].resource_type is status_models.ResourceType.storage + assert by_id["s3df-kubernetes"].resource_type is status_models.ResourceType.system + assert by_id["s3df-dtns"].resource_type is status_models.ResourceType.network + + +def test_registry_groups_match_yaml(): + by_id = S3DF_RESOURCES + assert by_id["s3df-ssh-bastions"].group == "access" + assert by_id["s3df-batch-servers"].group == "compute" + assert by_id["s3df-coact"].group == "accounts" + assert by_id["s3df-storage"].group == "storage" + assert by_id["s3df-dtns"].group == "data-transfer" + + +def test_parse_status_round_trip(): + assert parse_status("up") is status_models.Status.up + assert parse_status("down") is status_models.Status.down + assert parse_status("degraded") is status_models.Status.degraded + assert parse_status("unknown") is status_models.Status.unknown + assert parse_status(None) is None + assert parse_status("garbage") is status_models.Status.unknown + + +def test_site_id_is_non_empty(): + assert site_id() diff --git a/app/s3df/tests/test_user_lookup.py b/app/s3df/tests/test_user_lookup.py new file mode 100644 index 00000000..688edc4c --- /dev/null +++ b/app/s3df/tests/test_user_lookup.py @@ -0,0 +1,50 @@ +import argparse +import asyncio +import json +import logging +import os + +from app.s3df.clients.user_lookup import UserLookupClient + +logging.basicConfig(level=logging.INFO) +LOG = logging.getLogger(__name__) + + +def normalize_base_url(url: str) -> str: + base_url = url.rstrip("/") + if base_url.endswith("/graphql"): + base_url = base_url[: -len("/graphql")] + return base_url + + +async def main() -> int: + parser = argparse.ArgumentParser(description="Test the user-lookup client.") + parser.add_argument("username", help="Username to query") + parser.add_argument( + "--url", + default=os.getenv("USER_LOOKUP_URL"), + help="Base user-lookup URL. You can also set USER_LOOKUP_URL.", + ) + args = parser.parse_args() + + if not args.url: + LOG.error("Set USER_LOOKUP_URL or pass --url") + return 2 + + client = UserLookupClient(url=normalize_base_url(args.url)) + + try: + result = await client.get_user(args.username) + except ValueError as exc: + LOG.error("%s", exc) + return 1 + except Exception as exc: + LOG.exception("user-lookup request failed: %s", exc) + return 3 + + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) \ No newline at end of file