163 lines
6.9 KiB
Python
163 lines
6.9 KiB
Python
"""
|
|
Compound (multi-bot, time-sequenced) scenarios.
|
|
|
|
Each step is a tuple: (delay_seconds, target_key, scenario_name_or_None)
|
|
|
|
Delays are wall-clock seconds from when trigger() is called.
|
|
Fine-grained room sub-keys (hall-a/env, hall-a/pdu, etc.) are registered
|
|
by build_bots() in main.py so individual device classes can be targeted
|
|
without disturbing other bots in the same room.
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ── Scenario 1: Hot Night ────────────────────────────────────────────────────
|
|
# CRAC-01 compressor trips silently. The backup unit overworks itself.
|
|
# Rack temps climb, servers spin fans harder, power draw rises, smoke follows.
|
|
HOT_NIGHT = [
|
|
# t=0 CRAC-01 compressor trips — cooling capacity gone
|
|
( 0, "crac-01", "COOLING_FAILURE"),
|
|
# t=2m Rack inlet temps start climbing
|
|
( 120, "hall-a/env", "COOLING_FAILURE"),
|
|
# t=5m Servers spin fans faster → PDU draw rises across the room
|
|
( 300, "hall-a/pdu", "POWER_SPIKE"),
|
|
# t=8m CRAC-02 overworked — fan bearing begins degrading
|
|
( 480, "crac-02", "FAN_DEGRADATION"),
|
|
# t=10m Hot dust particles rising — VESDA alert fires
|
|
( 600, "vesda-hall-a", "VESDA_ALERT"),
|
|
]
|
|
|
|
|
|
# ── Scenario 2: Generator Test Gone Wrong ────────────────────────────────────
|
|
# Planned ATS transfer to generator. Generator was low on fuel (not checked).
|
|
# It runs for 10 minutes then faults. UPS must carry the full site load with
|
|
# no mains and no generator to recharge it.
|
|
GENERATOR_TEST_GONE_WRONG = [
|
|
# t=0 Utility fails — ATS transfers to generator
|
|
( 0, "ats-01", "ATS_TRANSFER"),
|
|
( 0, "gen-01", "GENERATOR_FAILURE"),
|
|
# t=10m Fuel critically low — wasn't topped up before the test
|
|
( 600, "gen-01", "GENERATOR_LOW_FUEL"),
|
|
# t=15m Generator faults — output collapses
|
|
( 900, "gen-01", "GENERATOR_FAULT"),
|
|
# t=16m ATS loses its only live feed — UPS is now the last line of defence
|
|
( 960, "ats-01", None),
|
|
( 960, "ups-01", "UPS_MAINS_FAILURE"),
|
|
( 960, "ups-02", "UPS_MAINS_FAILURE"),
|
|
]
|
|
|
|
|
|
# ── Scenario 3: Slow Burn ────────────────────────────────────────────────────
|
|
# A dirty filter nobody noticed. Airflow degrades for 30 simulated minutes.
|
|
# Temps creep, humidity climbs, VESDA alerts, then CRAC trips on protection.
|
|
SLOW_BURN = [
|
|
# t=0 Dirty filter — airflow begins throttling
|
|
( 0, "crac-01", "DIRTY_FILTER"),
|
|
# t=10m Rack temps and humidity start climbing together
|
|
( 600, "hall-a/env", "SLOW_BURN"),
|
|
# t=25m Particulates agitated by heat — VESDA alert fires
|
|
( 1500, "vesda-hall-a", "VESDA_ALERT"),
|
|
# t=30m CRAC-01 protection trips on high return-air temperature
|
|
( 1800, "crac-01", "COOLING_FAILURE"),
|
|
]
|
|
|
|
|
|
# ── Scenario 4: Last Resort ──────────────────────────────────────────────────
|
|
# Utility fails. Generator starts. Generator fails after 2 minutes.
|
|
# UPS absorbs full load with no recharge path. Load spikes on inrush.
|
|
# UPS components overheat. VESDA escalates to fire.
|
|
LAST_RESORT = [
|
|
# t=0 Utility power lost — ATS transfers to generator
|
|
( 0, "ats-01", "ATS_TRANSFER"),
|
|
( 0, "gen-01", "GENERATOR_FAILURE"),
|
|
# t=2m Generator fails to sustain load — faults out
|
|
( 120, "gen-01", "GENERATOR_FAULT"),
|
|
# t=2m30s ATS loses generator feed — UPS is the only thing left
|
|
( 150, "ats-01", None),
|
|
( 155, "ups-01", "UPS_OVERLOAD"),
|
|
( 155, "ups-02", "UPS_OVERLOAD"),
|
|
# t=3m PDU inrush spike as UPS absorbs full site load
|
|
( 180, "hall-a/pdu", "POWER_SPIKE"),
|
|
( 180, "hall-b/pdu", "POWER_SPIKE"),
|
|
# t=6m UPS rectifiers heating — VESDA detects particles
|
|
( 360, "vesda-hall-a", "VESDA_ALERT"),
|
|
( 380, "vesda-hall-b", "VESDA_ALERT"),
|
|
# t=9m Thermal runaway — VESDA escalates to fire
|
|
( 540, "vesda-hall-a", "VESDA_FIRE"),
|
|
( 570, "vesda-hall-b", "VESDA_FIRE"),
|
|
]
|
|
|
|
|
|
COMPOUND_SCENARIOS: dict[str, list[tuple]] = {
|
|
"HOT_NIGHT": HOT_NIGHT,
|
|
"GENERATOR_TEST_GONE_WRONG": GENERATOR_TEST_GONE_WRONG,
|
|
"SLOW_BURN": SLOW_BURN,
|
|
"LAST_RESORT": LAST_RESORT,
|
|
}
|
|
|
|
|
|
class CompoundOrchestrator:
|
|
"""
|
|
Fires multi-bot, time-sequenced scenarios.
|
|
|
|
Steps are scheduled from the moment trigger() is called. Any currently
|
|
running scenario is cancelled before the new one starts.
|
|
"""
|
|
|
|
def __init__(self, by_key: dict) -> None:
|
|
self._by_key = by_key
|
|
self._active: asyncio.Task | None = None
|
|
|
|
def trigger(self, name: str) -> None:
|
|
steps = COMPOUND_SCENARIOS.get(name)
|
|
if steps is None:
|
|
logger.warning(f"[Compound] Unknown scenario: '{name}'")
|
|
return
|
|
self._cancel_active()
|
|
self._active = asyncio.create_task(self._run(name, steps))
|
|
|
|
def reset(self) -> None:
|
|
self._cancel_active()
|
|
seen: set = set()
|
|
for v in self._by_key.values():
|
|
targets = v if isinstance(v, list) else [v]
|
|
for bot in targets:
|
|
if id(bot) not in seen:
|
|
bot.set_scenario(None)
|
|
seen.add(id(bot))
|
|
|
|
def _cancel_active(self) -> None:
|
|
if self._active and not self._active.done():
|
|
self._active.cancel()
|
|
self._active = None
|
|
|
|
async def _run(self, name: str, steps: list[tuple]) -> None:
|
|
logger.info(f"[Compound] '{name}' started — {len(steps)} steps")
|
|
sorted_steps = sorted(steps, key=lambda s: s[0])
|
|
t0 = asyncio.get_event_loop().time()
|
|
try:
|
|
for delay, target_key, scenario in sorted_steps:
|
|
elapsed = asyncio.get_event_loop().time() - t0
|
|
wait = delay - elapsed
|
|
if wait > 0:
|
|
await asyncio.sleep(wait)
|
|
self._apply(target_key, scenario, delay)
|
|
logger.info(f"[Compound] '{name}' complete")
|
|
except asyncio.CancelledError:
|
|
logger.info(f"[Compound] '{name}' cancelled")
|
|
|
|
def _apply(self, target_key: str, scenario: str | None, delay: int) -> None:
|
|
entry = self._by_key.get(target_key)
|
|
if entry is None:
|
|
logger.warning(f"[Compound] Unknown target '{target_key}' — skipping")
|
|
return
|
|
targets = entry if isinstance(entry, list) else [entry]
|
|
for bot in targets:
|
|
bot.set_scenario(scenario)
|
|
logger.info(
|
|
f"[Compound] t+{delay}s {target_key} ({len(targets)} bots)"
|
|
f" → {scenario or 'RESET'}"
|
|
)
|