""" Compound (multi-bot, time-sequenced) scenarios. Each step is a tuple: (delay_seconds, target_key, scenario_name_or_None) Delays are wall-clock seconds from when trigger() is called. Fine-grained room sub-keys (hall-a/env, hall-a/pdu, etc.) are registered by build_bots() in main.py so individual device classes can be targeted without disturbing other bots in the same room. """ import asyncio import logging logger = logging.getLogger(__name__) # ── Scenario 1: Hot Night ──────────────────────────────────────────────────── # CRAC-01 compressor trips silently. The backup unit overworks itself. # Rack temps climb, servers spin fans harder, power draw rises, smoke follows. HOT_NIGHT = [ # t=0 CRAC-01 compressor trips — cooling capacity gone ( 0, "crac-01", "COOLING_FAILURE"), # t=2m Rack inlet temps start climbing ( 120, "hall-a/env", "COOLING_FAILURE"), # t=5m Servers spin fans faster → PDU draw rises across the room ( 300, "hall-a/pdu", "POWER_SPIKE"), # t=8m CRAC-02 overworked — fan bearing begins degrading ( 480, "crac-02", "FAN_DEGRADATION"), # t=10m Hot dust particles rising — VESDA alert fires ( 600, "vesda-hall-a", "VESDA_ALERT"), ] # ── Scenario 2: Generator Test Gone Wrong ──────────────────────────────────── # Planned ATS transfer to generator. Generator was low on fuel (not checked). # It runs for 10 minutes then faults. UPS must carry the full site load with # no mains and no generator to recharge it. GENERATOR_TEST_GONE_WRONG = [ # t=0 Utility fails — ATS transfers to generator ( 0, "ats-01", "ATS_TRANSFER"), ( 0, "gen-01", "GENERATOR_FAILURE"), # t=10m Fuel critically low — wasn't topped up before the test ( 600, "gen-01", "GENERATOR_LOW_FUEL"), # t=15m Generator faults — output collapses ( 900, "gen-01", "GENERATOR_FAULT"), # t=16m ATS loses its only live feed — UPS is now the last line of defence ( 960, "ats-01", None), ( 960, "ups-01", "UPS_MAINS_FAILURE"), ( 960, "ups-02", "UPS_MAINS_FAILURE"), ] # ── Scenario 3: Slow Burn ──────────────────────────────────────────────────── # A dirty filter nobody noticed. Airflow degrades for 30 simulated minutes. # Temps creep, humidity climbs, VESDA alerts, then CRAC trips on protection. SLOW_BURN = [ # t=0 Dirty filter — airflow begins throttling ( 0, "crac-01", "DIRTY_FILTER"), # t=10m Rack temps and humidity start climbing together ( 600, "hall-a/env", "SLOW_BURN"), # t=25m Particulates agitated by heat — VESDA alert fires ( 1500, "vesda-hall-a", "VESDA_ALERT"), # t=30m CRAC-01 protection trips on high return-air temperature ( 1800, "crac-01", "COOLING_FAILURE"), ] # ── Scenario 4: Last Resort ────────────────────────────────────────────────── # Utility fails. Generator starts. Generator fails after 2 minutes. # UPS absorbs full load with no recharge path. Load spikes on inrush. # UPS components overheat. VESDA escalates to fire. LAST_RESORT = [ # t=0 Utility power lost — ATS transfers to generator ( 0, "ats-01", "ATS_TRANSFER"), ( 0, "gen-01", "GENERATOR_FAILURE"), # t=2m Generator fails to sustain load — faults out ( 120, "gen-01", "GENERATOR_FAULT"), # t=2m30s ATS loses generator feed — UPS is the only thing left ( 150, "ats-01", None), ( 155, "ups-01", "UPS_OVERLOAD"), ( 155, "ups-02", "UPS_OVERLOAD"), # t=3m PDU inrush spike as UPS absorbs full site load ( 180, "hall-a/pdu", "POWER_SPIKE"), ( 180, "hall-b/pdu", "POWER_SPIKE"), # t=6m UPS rectifiers heating — VESDA detects particles ( 360, "vesda-hall-a", "VESDA_ALERT"), ( 380, "vesda-hall-b", "VESDA_ALERT"), # t=9m Thermal runaway — VESDA escalates to fire ( 540, "vesda-hall-a", "VESDA_FIRE"), ( 570, "vesda-hall-b", "VESDA_FIRE"), ] COMPOUND_SCENARIOS: dict[str, list[tuple]] = { "HOT_NIGHT": HOT_NIGHT, "GENERATOR_TEST_GONE_WRONG": GENERATOR_TEST_GONE_WRONG, "SLOW_BURN": SLOW_BURN, "LAST_RESORT": LAST_RESORT, } class CompoundOrchestrator: """ Fires multi-bot, time-sequenced scenarios. Steps are scheduled from the moment trigger() is called. Any currently running scenario is cancelled before the new one starts. """ def __init__(self, by_key: dict) -> None: self._by_key = by_key self._active: asyncio.Task | None = None def trigger(self, name: str) -> None: steps = COMPOUND_SCENARIOS.get(name) if steps is None: logger.warning(f"[Compound] Unknown scenario: '{name}'") return self._cancel_active() self._active = asyncio.create_task(self._run(name, steps)) def reset(self) -> None: self._cancel_active() seen: set = set() for v in self._by_key.values(): targets = v if isinstance(v, list) else [v] for bot in targets: if id(bot) not in seen: bot.set_scenario(None) seen.add(id(bot)) def _cancel_active(self) -> None: if self._active and not self._active.done(): self._active.cancel() self._active = None async def _run(self, name: str, steps: list[tuple]) -> None: logger.info(f"[Compound] '{name}' started — {len(steps)} steps") sorted_steps = sorted(steps, key=lambda s: s[0]) t0 = asyncio.get_event_loop().time() try: for delay, target_key, scenario in sorted_steps: elapsed = asyncio.get_event_loop().time() - t0 wait = delay - elapsed if wait > 0: await asyncio.sleep(wait) self._apply(target_key, scenario, delay) logger.info(f"[Compound] '{name}' complete") except asyncio.CancelledError: logger.info(f"[Compound] '{name}' cancelled") def _apply(self, target_key: str, scenario: str | None, delay: int) -> None: entry = self._by_key.get(target_key) if entry is None: logger.warning(f"[Compound] Unknown target '{target_key}' — skipping") return targets = entry if isinstance(entry, list) else [entry] for bot in targets: bot.set_scenario(scenario) logger.info( f"[Compound] t+{delay}s {target_key} ({len(targets)} bots)" f" → {scenario or 'RESET'}" )