first commit
This commit is contained in:
commit
4b98219bf7
144 changed files with 31561 additions and 0 deletions
0
backend/services/__init__.py
Normal file
0
backend/services/__init__.py
Normal file
152
backend/services/alarm_engine.py
Normal file
152
backend/services/alarm_engine.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import logging
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── In-memory threshold cache ──────────────────────────────────────────────────
|
||||
# Loaded from DB on first use; invalidated by settings API after updates.
|
||||
# Falls back to hard-coded defaults if DB has no rows yet (pre-seed).
|
||||
|
||||
_caches: dict[str, list[dict]] = {}
|
||||
_dirty_sites: set[str] = {"sg-01"} # start dirty so first request loads from DB
|
||||
|
||||
|
||||
def invalidate_threshold_cache(site_id: str = "sg-01") -> None:
|
||||
"""Mark a site's cache as stale. Called by settings API after threshold changes."""
|
||||
_dirty_sites.add(site_id)
|
||||
|
||||
|
||||
async def _ensure_cache(session: AsyncSession, site_id: str) -> None:
|
||||
if site_id not in _dirty_sites and site_id in _caches:
|
||||
return
|
||||
|
||||
result = await session.execute(text("""
|
||||
SELECT sensor_type, threshold_value, direction, severity, message_template
|
||||
FROM alarm_thresholds
|
||||
WHERE site_id = :site_id AND enabled = true
|
||||
ORDER BY id
|
||||
"""), {"site_id": site_id})
|
||||
rows = result.mappings().all()
|
||||
|
||||
if rows:
|
||||
_caches[site_id] = [dict(r) for r in rows]
|
||||
else:
|
||||
# DB not yet seeded — fall back to hard-coded defaults
|
||||
_caches[site_id] = _FALLBACK_RULES
|
||||
|
||||
_dirty_sites.discard(site_id)
|
||||
logger.info(f"Loaded {len(_caches[site_id])} threshold rules for {site_id}")
|
||||
|
||||
|
||||
async def check_and_update_alarms(
|
||||
session: AsyncSession,
|
||||
sensor_id: str,
|
||||
sensor_type: str,
|
||||
site_id: str,
|
||||
room_id: str | None,
|
||||
rack_id: str | None,
|
||||
value: float,
|
||||
) -> None:
|
||||
await _ensure_cache(session, site_id)
|
||||
|
||||
for rule in _caches.get(site_id, []):
|
||||
if rule["sensor_type"] != sensor_type:
|
||||
continue
|
||||
|
||||
threshold = rule["threshold_value"]
|
||||
direction = rule["direction"]
|
||||
severity = rule["severity"]
|
||||
msg_tpl = rule["message_template"]
|
||||
|
||||
breached = (
|
||||
(direction == "above" and value > threshold) or
|
||||
(direction == "below" and value < threshold)
|
||||
)
|
||||
|
||||
if breached:
|
||||
existing = await session.execute(text("""
|
||||
SELECT id FROM alarms
|
||||
WHERE sensor_id = :sid AND severity = :sev AND state = 'active'
|
||||
LIMIT 1
|
||||
"""), {"sid": sensor_id, "sev": severity})
|
||||
|
||||
if not existing.fetchone():
|
||||
message = msg_tpl.format(value=value, sensor_id=sensor_id)
|
||||
await session.execute(text("""
|
||||
INSERT INTO alarms
|
||||
(sensor_id, site_id, room_id, rack_id, severity, message, state, triggered_at)
|
||||
VALUES
|
||||
(:sensor_id, :site_id, :room_id, :rack_id, :severity, :message, 'active', NOW())
|
||||
"""), {
|
||||
"sensor_id": sensor_id, "site_id": site_id,
|
||||
"room_id": room_id, "rack_id": rack_id,
|
||||
"severity": severity, "message": message,
|
||||
})
|
||||
logger.info(f"Alarm raised [{severity}]: {message}")
|
||||
else:
|
||||
await session.execute(text("""
|
||||
UPDATE alarms
|
||||
SET state = 'resolved', resolved_at = NOW()
|
||||
WHERE sensor_id = :sid AND severity = :sev AND state = 'active'
|
||||
"""), {"sid": sensor_id, "sev": severity})
|
||||
|
||||
|
||||
# ── Hard-coded fallback (used before DB seed runs) ─────────────────────────────
|
||||
|
||||
_FALLBACK_RULES: list[dict] = [
|
||||
{"sensor_type": st, "threshold_value": tv, "direction": d, "severity": s, "message_template": m}
|
||||
for st, tv, d, s, m in [
|
||||
("temperature", 28.0, "above", "warning", "Temperature elevated at {sensor_id}: {value:.1f}°C"),
|
||||
("temperature", 32.0, "above", "critical", "Temperature critical at {sensor_id}: {value:.1f}°C"),
|
||||
("humidity", 65.0, "above", "warning", "Humidity elevated at {sensor_id}: {value:.0f}%"),
|
||||
("power_kw", 7.5, "above", "warning", "PDU load elevated at {sensor_id}: {value:.1f} kW"),
|
||||
("power_kw", 9.5, "above", "critical", "PDU load critical at {sensor_id}: {value:.1f} kW"),
|
||||
("ups_charge", 80.0, "below", "warning", "UPS battery low at {sensor_id}: {value:.0f}%"),
|
||||
("ups_charge", 50.0, "below", "critical", "UPS battery critical at {sensor_id}: {value:.0f}%"),
|
||||
("ups_state", 0.5, "above", "critical", "UPS switched to battery at {sensor_id} — mains power lost"),
|
||||
("ups_state", 1.5, "above", "critical", "UPS overloaded at {sensor_id} — immediate risk of failure"),
|
||||
("ups_load", 85.0, "above", "warning", "UPS load high at {sensor_id}: {value:.0f}%"),
|
||||
("ups_load", 95.0, "above", "critical", "UPS load critical at {sensor_id}: {value:.0f}% — overload"),
|
||||
("ups_runtime", 15.0, "below", "warning", "UPS runtime low at {sensor_id}: {value:.0f} min remaining"),
|
||||
("ups_runtime", 5.0, "below", "critical", "UPS runtime critical at {sensor_id}: {value:.0f} min — imminent shutdown"),
|
||||
("leak", 0.5, "above", "critical", "Water leak detected at {sensor_id}!"),
|
||||
("cooling_cap_pct", 90.0, "above", "warning", "CRAC near capacity limit at {sensor_id}: {value:.1f}%"),
|
||||
("cooling_cop", 1.5, "below", "warning", "CRAC running inefficiently at {sensor_id}: COP {value:.2f}"),
|
||||
("cooling_comp_load", 95.0, "above", "warning", "CRAC compressor overloaded at {sensor_id}: {value:.1f}%"),
|
||||
("cooling_high_press", 22.0, "above", "critical", "CRAC high refrigerant pressure at {sensor_id}: {value:.1f} bar"),
|
||||
("cooling_low_press", 3.0, "below", "critical", "CRAC low refrigerant pressure at {sensor_id}: {value:.1f} bar — possible leak"),
|
||||
("cooling_superheat", 16.0, "above", "warning", "CRAC discharge superheat high at {sensor_id}: {value:.1f}°C"),
|
||||
("cooling_filter_dp", 80.0, "above", "warning", "CRAC filter requires attention at {sensor_id}: {value:.0f} Pa"),
|
||||
("cooling_filter_dp", 120.0, "above", "critical", "CRAC filter critically blocked at {sensor_id}: {value:.0f} Pa — replace now"),
|
||||
("cooling_return", 36.0, "above", "warning", "CRAC return air temperature high at {sensor_id}: {value:.1f}°C"),
|
||||
("cooling_return", 42.0, "above", "critical", "CRAC return air temperature critical at {sensor_id}: {value:.1f}°C"),
|
||||
("gen_fuel_pct", 25.0, "below", "warning", "Generator fuel low at {sensor_id}: {value:.1f}%"),
|
||||
("gen_fuel_pct", 10.0, "below", "critical", "Generator fuel critical at {sensor_id}: {value:.1f}%"),
|
||||
("gen_state", 0.5, "above", "warning", "Generator running at {sensor_id} — site is on standby power"),
|
||||
("gen_state", -0.5, "below", "critical", "Generator fault at {sensor_id} — no standby power available"),
|
||||
("gen_load_pct", 85.0, "above", "warning", "Generator load high at {sensor_id}: {value:.1f}%"),
|
||||
("gen_load_pct", 95.0, "above", "critical", "Generator overloaded at {sensor_id}: {value:.1f}%"),
|
||||
("gen_coolant_c", 95.0, "above", "warning", "Generator coolant temperature high at {sensor_id}: {value:.1f}°C"),
|
||||
("gen_coolant_c", 105.0, "above", "critical", "Generator coolant critical at {sensor_id}: {value:.1f}°C — risk of shutdown"),
|
||||
("gen_oil_press", 2.0, "below", "critical", "Generator oil pressure low at {sensor_id}: {value:.1f} bar"),
|
||||
("pdu_imbalance", 5.0, "above", "warning", "PDU phase imbalance at {sensor_id}: {value:.1f}%"),
|
||||
("pdu_imbalance", 15.0, "above", "critical", "PDU phase imbalance critical at {sensor_id}: {value:.1f}%"),
|
||||
("ats_active", 1.5, "above", "warning", "ATS transferred to generator at {sensor_id} — utility power lost"),
|
||||
("ats_ua_v", 50.0, "below", "critical", "Utility A power failure at {sensor_id} — supply lost"),
|
||||
("chiller_state", 0.5, "below", "critical", "Chiller fault at {sensor_id} — CHW supply lost"),
|
||||
("chiller_cop", 2.5, "below", "warning", "Chiller running inefficiently at {sensor_id}: COP {value:.2f}"),
|
||||
("vesda_level", 0.5, "above", "warning", "VESDA smoke detected at {sensor_id}: level elevated"),
|
||||
("vesda_level", 1.5, "above", "warning", "VESDA action threshold reached at {sensor_id}"),
|
||||
("vesda_level", 2.5, "above", "critical", "VESDA FIRE ALARM at {sensor_id}!"),
|
||||
("vesda_flow", 0.5, "below", "critical", "VESDA aspirator flow fault at {sensor_id} — detector may be compromised"),
|
||||
("vesda_det1", 0.5, "below", "warning", "VESDA detector 1 fault at {sensor_id}"),
|
||||
("vesda_det2", 0.5, "below", "warning", "VESDA detector 2 fault at {sensor_id}"),
|
||||
("net_state", 0.5, "above", "warning", "Network switch degraded at {sensor_id}"),
|
||||
("net_state", 1.5, "above", "critical", "Network switch down at {sensor_id} — connectivity lost"),
|
||||
("net_pkt_loss_pct", 1.0, "above", "warning", "Packet loss detected at {sensor_id}: {value:.1f}%"),
|
||||
("net_pkt_loss_pct", 5.0, "above", "critical", "High packet loss at {sensor_id}: {value:.1f}%"),
|
||||
("net_temp_c", 65.0, "above", "warning", "Switch temperature high at {sensor_id}: {value:.1f}°C"),
|
||||
("net_temp_c", 75.0, "above", "critical", "Switch temperature critical at {sensor_id}: {value:.1f}°C"),
|
||||
]
|
||||
]
|
||||
328
backend/services/mqtt_subscriber.py
Normal file
328
backend/services/mqtt_subscriber.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import aiomqtt
|
||||
from sqlalchemy import text
|
||||
|
||||
from core.config import settings
|
||||
from core.database import AsyncSessionLocal
|
||||
from services.alarm_engine import check_and_update_alarms
|
||||
from services.ws_manager import manager as ws_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_topic(topic: str) -> dict | None:
|
||||
"""
|
||||
Topic formats:
|
||||
bms/{site_id}/{room_id}/{rack_id}/env — rack environment
|
||||
bms/{site_id}/{room_id}/{rack_id}/power — rack PDU power
|
||||
bms/{site_id}/cooling/{crac_id} — CRAC unit
|
||||
bms/{site_id}/cooling/chiller/{chiller_id} — chiller plant
|
||||
bms/{site_id}/power/{ups_id} — UPS unit
|
||||
bms/{site_id}/power/ats/{ats_id} — ATS transfer switch
|
||||
bms/{site_id}/generator/{gen_id} — diesel generator
|
||||
bms/{site_id}/fire/{zone_id} — VESDA fire zone
|
||||
bms/{site_id}/leak/{sensor_id} — water leak sensor
|
||||
"""
|
||||
parts = topic.split("/")
|
||||
if len(parts) < 4 or parts[0] != "bms":
|
||||
return None
|
||||
|
||||
site_id = parts[1]
|
||||
|
||||
# 5-part: rack env/power OR cooling/chiller/{id} OR power/ats/{id}
|
||||
if len(parts) == 5:
|
||||
if parts[4] in ("env", "power"):
|
||||
return {
|
||||
"site_id": site_id, "room_id": parts[2],
|
||||
"rack_id": parts[3], "device_id": None, "msg_type": parts[4],
|
||||
}
|
||||
if parts[2] == "cooling" and parts[3] == "chiller":
|
||||
return {
|
||||
"site_id": site_id, "room_id": None, "rack_id": None,
|
||||
"device_id": parts[4], "msg_type": "chiller",
|
||||
}
|
||||
if parts[2] == "power" and parts[3] == "ats":
|
||||
return {
|
||||
"site_id": site_id, "room_id": None, "rack_id": None,
|
||||
"device_id": parts[4], "msg_type": "ats",
|
||||
}
|
||||
|
||||
# 4-part: bms/{site_id}/{room_id}/particles
|
||||
if len(parts) == 4 and parts[3] == "particles":
|
||||
return {
|
||||
"site_id": site_id, "room_id": parts[2], "rack_id": None,
|
||||
"device_id": None, "msg_type": "particles",
|
||||
}
|
||||
|
||||
# 4-part: known subsystem topics
|
||||
if len(parts) == 4 and parts[2] in ("cooling", "power", "leak", "generator", "fire", "network"):
|
||||
return {
|
||||
"site_id": site_id, "room_id": None, "rack_id": None,
|
||||
"device_id": parts[3], "msg_type": parts[2],
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
async def process_message(topic: str, payload: dict) -> None:
|
||||
meta = parse_topic(topic)
|
||||
if not meta:
|
||||
return
|
||||
|
||||
site_id = meta["site_id"]
|
||||
room_id = meta["room_id"]
|
||||
rack_id = meta["rack_id"]
|
||||
device_id = meta["device_id"]
|
||||
msg_type = meta["msg_type"]
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Build list of (sensor_id, sensor_type, value, unit) tuples
|
||||
readings: list[tuple[str, str, float, str]] = []
|
||||
|
||||
if msg_type == "env" and rack_id:
|
||||
base = f"{site_id}/{room_id}/{rack_id}"
|
||||
if "temperature" in payload:
|
||||
readings.append((f"{base}/temperature", "temperature", float(payload["temperature"]), "°C"))
|
||||
if "humidity" in payload:
|
||||
readings.append((f"{base}/humidity", "humidity", float(payload["humidity"]), "%"))
|
||||
|
||||
elif msg_type == "power" and rack_id:
|
||||
base = f"{site_id}/{room_id}/{rack_id}"
|
||||
if "load_kw" in payload:
|
||||
readings.append((f"{base}/power_kw", "power_kw", float(payload["load_kw"]), "kW"))
|
||||
# Per-phase PDU data
|
||||
for key, s_type, unit in [
|
||||
("phase_a_kw", "pdu_phase_a_kw", "kW"),
|
||||
("phase_b_kw", "pdu_phase_b_kw", "kW"),
|
||||
("phase_c_kw", "pdu_phase_c_kw", "kW"),
|
||||
("phase_a_a", "pdu_phase_a_a", "A"),
|
||||
("phase_b_a", "pdu_phase_b_a", "A"),
|
||||
("phase_c_a", "pdu_phase_c_a", "A"),
|
||||
("imbalance_pct", "pdu_imbalance", "%"),
|
||||
]:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
|
||||
elif msg_type == "cooling" and device_id:
|
||||
base = f"{site_id}/cooling/{device_id}"
|
||||
crac_fields = [
|
||||
# (payload_key, sensor_type, unit)
|
||||
("supply_temp", "cooling_supply", "°C"),
|
||||
("return_temp", "cooling_return", "°C"),
|
||||
("fan_pct", "cooling_fan", "%"),
|
||||
("supply_humidity", "cooling_supply_hum", "%"),
|
||||
("return_humidity", "cooling_return_hum", "%"),
|
||||
("airflow_cfm", "cooling_airflow", "CFM"),
|
||||
("filter_dp_pa", "cooling_filter_dp", "Pa"),
|
||||
("cooling_capacity_kw", "cooling_cap_kw", "kW"),
|
||||
("cooling_capacity_pct", "cooling_cap_pct", "%"),
|
||||
("cop", "cooling_cop", ""),
|
||||
("sensible_heat_ratio", "cooling_shr", ""),
|
||||
("compressor_state", "cooling_comp_state", ""),
|
||||
("compressor_load_pct", "cooling_comp_load", "%"),
|
||||
("compressor_power_kw", "cooling_comp_power", "kW"),
|
||||
("compressor_run_hours", "cooling_comp_hours", "h"),
|
||||
("high_pressure_bar", "cooling_high_press", "bar"),
|
||||
("low_pressure_bar", "cooling_low_press", "bar"),
|
||||
("discharge_superheat_c", "cooling_superheat", "°C"),
|
||||
("liquid_subcooling_c", "cooling_subcooling", "°C"),
|
||||
("fan_rpm", "cooling_fan_rpm", "RPM"),
|
||||
("fan_power_kw", "cooling_fan_power", "kW"),
|
||||
("fan_run_hours", "cooling_fan_hours", "h"),
|
||||
("total_unit_power_kw", "cooling_unit_power", "kW"),
|
||||
("input_voltage_v", "cooling_voltage", "V"),
|
||||
("input_current_a", "cooling_current", "A"),
|
||||
("power_factor", "cooling_pf", ""),
|
||||
]
|
||||
for key, s_type, unit in crac_fields:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
|
||||
elif msg_type == "power" and device_id:
|
||||
base = f"{site_id}/power/{device_id}"
|
||||
for key, s_type, unit in [
|
||||
("charge_pct", "ups_charge", "%"),
|
||||
("load_pct", "ups_load", "%"),
|
||||
("runtime_min", "ups_runtime", "min"),
|
||||
("voltage", "ups_voltage", "V"),
|
||||
]:
|
||||
if key in payload:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
# Store state explicitly: 0.0 = online, 1.0 = on_battery, 2.0 = overload
|
||||
if "state" in payload:
|
||||
state_val = {"online": 0.0, "on_battery": 1.0, "overload": 2.0}.get(payload["state"], 0.0)
|
||||
readings.append((f"{base}/state", "ups_state", state_val, ""))
|
||||
|
||||
elif msg_type == "generator" and device_id:
|
||||
base = f"{site_id}/generator/{device_id}"
|
||||
state_map = {"standby": 0.0, "running": 1.0, "test": 2.0, "fault": -1.0}
|
||||
for key, s_type, unit in [
|
||||
("fuel_pct", "gen_fuel_pct", "%"),
|
||||
("fuel_litres", "gen_fuel_l", "L"),
|
||||
("fuel_rate_lph", "gen_fuel_rate", "L/h"),
|
||||
("load_kw", "gen_load_kw", "kW"),
|
||||
("load_pct", "gen_load_pct", "%"),
|
||||
("run_hours", "gen_run_hours", "h"),
|
||||
("voltage_v", "gen_voltage_v", "V"),
|
||||
("frequency_hz", "gen_freq_hz", "Hz"),
|
||||
("engine_rpm", "gen_rpm", "RPM"),
|
||||
("oil_pressure_bar", "gen_oil_press", "bar"),
|
||||
("coolant_temp_c", "gen_coolant_c", "°C"),
|
||||
("exhaust_temp_c", "gen_exhaust_c", "°C"),
|
||||
("alternator_temp_c", "gen_alt_temp_c", "°C"),
|
||||
("power_factor", "gen_pf", ""),
|
||||
("battery_v", "gen_batt_v", "V"),
|
||||
]:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
if "state" in payload:
|
||||
readings.append((f"{base}/state", "gen_state", state_map.get(payload["state"], 0.0), ""))
|
||||
|
||||
elif msg_type == "ats" and device_id:
|
||||
base = f"{site_id}/power/ats/{device_id}"
|
||||
feed_map = {"utility-a": 0.0, "utility-b": 1.0, "generator": 2.0}
|
||||
for key, s_type, unit in [
|
||||
("transfer_count", "ats_xfer_count", ""),
|
||||
("last_transfer_ms", "ats_xfer_ms", "ms"),
|
||||
("utility_a_v", "ats_ua_v", "V"),
|
||||
("utility_b_v", "ats_ub_v", "V"),
|
||||
("generator_v", "ats_gen_v", "V"),
|
||||
]:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
if "active_feed" in payload:
|
||||
readings.append((f"{base}/active_feed", "ats_active",
|
||||
feed_map.get(payload["active_feed"], 0.0), ""))
|
||||
if "state" in payload:
|
||||
readings.append((f"{base}/state", "ats_state",
|
||||
1.0 if payload["state"] == "transferring" else 0.0, ""))
|
||||
|
||||
elif msg_type == "chiller" and device_id:
|
||||
base = f"{site_id}/cooling/chiller/{device_id}"
|
||||
for key, s_type, unit in [
|
||||
("chw_supply_c", "chiller_chw_supply", "°C"),
|
||||
("chw_return_c", "chiller_chw_return", "°C"),
|
||||
("chw_delta_c", "chiller_chw_delta", "°C"),
|
||||
("flow_gpm", "chiller_flow_gpm", "GPM"),
|
||||
("cooling_load_kw", "chiller_load_kw", "kW"),
|
||||
("cooling_load_pct", "chiller_load_pct", "%"),
|
||||
("cop", "chiller_cop", ""),
|
||||
("compressor_load_pct", "chiller_comp_load", "%"),
|
||||
("condenser_pressure_bar", "chiller_cond_press", "bar"),
|
||||
("evaporator_pressure_bar", "chiller_evap_press", "bar"),
|
||||
("cw_supply_c", "chiller_cw_supply", "°C"),
|
||||
("cw_return_c", "chiller_cw_return", "°C"),
|
||||
("run_hours", "chiller_run_hours", "h"),
|
||||
]:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
if "state" in payload:
|
||||
readings.append((f"{base}/state", "chiller_state",
|
||||
1.0 if payload["state"] == "online" else 0.0, ""))
|
||||
|
||||
elif msg_type == "fire" and device_id:
|
||||
base = f"{site_id}/fire/{device_id}"
|
||||
level_map = {"normal": 0.0, "alert": 1.0, "action": 2.0, "fire": 3.0}
|
||||
if "level" in payload:
|
||||
readings.append((f"{base}/level", "vesda_level",
|
||||
level_map.get(payload["level"], 0.0), ""))
|
||||
if "obscuration_pct_m" in payload:
|
||||
readings.append((f"{base}/obscuration", "vesda_obscuration",
|
||||
float(payload["obscuration_pct_m"]), "%/m"))
|
||||
for key, s_type in [
|
||||
("detector_1_ok", "vesda_det1"),
|
||||
("detector_2_ok", "vesda_det2"),
|
||||
("power_ok", "vesda_power"),
|
||||
("flow_ok", "vesda_flow"),
|
||||
]:
|
||||
if key in payload:
|
||||
readings.append((f"{base}/{key}", s_type,
|
||||
1.0 if payload[key] else 0.0, ""))
|
||||
|
||||
elif msg_type == "network" and device_id:
|
||||
base = f"{site_id}/network/{device_id}"
|
||||
state_map = {"up": 0.0, "degraded": 1.0, "down": 2.0}
|
||||
for key, s_type, unit in [
|
||||
("uptime_s", "net_uptime_s", "s"),
|
||||
("active_ports", "net_active_ports", ""),
|
||||
("bandwidth_in_mbps", "net_bw_in_mbps", "Mbps"),
|
||||
("bandwidth_out_mbps","net_bw_out_mbps", "Mbps"),
|
||||
("cpu_pct", "net_cpu_pct", "%"),
|
||||
("mem_pct", "net_mem_pct", "%"),
|
||||
("temperature_c", "net_temp_c", "°C"),
|
||||
("packet_loss_pct", "net_pkt_loss_pct", "%"),
|
||||
]:
|
||||
if payload.get(key) is not None:
|
||||
readings.append((f"{base}/{key}", s_type, float(payload[key]), unit))
|
||||
if "state" in payload:
|
||||
readings.append((f"{base}/state", "net_state",
|
||||
state_map.get(payload["state"], 0.0), ""))
|
||||
|
||||
elif msg_type == "leak" and device_id:
|
||||
state = payload.get("state", "clear")
|
||||
readings.append((
|
||||
f"{site_id}/leak/{device_id}", "leak",
|
||||
1.0 if state == "detected" else 0.0, "",
|
||||
))
|
||||
|
||||
elif msg_type == "particles":
|
||||
base = f"{site_id}/{room_id}/particles"
|
||||
if "particles_0_5um" in payload:
|
||||
readings.append((f"{base}/0_5um", "particles_0_5um", float(payload["particles_0_5um"]), "/m³"))
|
||||
if "particles_5um" in payload:
|
||||
readings.append((f"{base}/5um", "particles_5um", float(payload["particles_5um"]), "/m³"))
|
||||
|
||||
if not readings:
|
||||
return
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
for sensor_id, sensor_type, value, unit in readings:
|
||||
await session.execute(text("""
|
||||
INSERT INTO readings
|
||||
(recorded_at, sensor_id, sensor_type, site_id, room_id, rack_id, value, unit)
|
||||
VALUES
|
||||
(:ts, :sid, :stype, :site, :room, :rack, :val, :unit)
|
||||
"""), {
|
||||
"ts": now, "sid": sensor_id, "stype": sensor_type,
|
||||
"site": site_id, "room": room_id, "rack": rack_id,
|
||||
"val": value, "unit": unit,
|
||||
})
|
||||
await check_and_update_alarms(
|
||||
session, sensor_id, sensor_type, site_id, room_id, rack_id, value
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# Push to any connected WebSocket clients
|
||||
await ws_manager.broadcast({
|
||||
"topic": topic,
|
||||
"site_id": site_id,
|
||||
"room_id": room_id,
|
||||
"rack_id": rack_id,
|
||||
"readings": [
|
||||
{"sensor_id": s, "type": t, "value": v, "unit": u}
|
||||
for s, t, v, u in readings
|
||||
],
|
||||
"timestamp": now.isoformat(),
|
||||
})
|
||||
|
||||
|
||||
async def run_subscriber() -> None:
|
||||
"""Runs forever, reconnecting on any failure."""
|
||||
while True:
|
||||
try:
|
||||
logger.info(f"Connecting to MQTT at {settings.MQTT_HOST}:{settings.MQTT_PORT}")
|
||||
async with aiomqtt.Client(settings.MQTT_HOST, port=settings.MQTT_PORT) as client:
|
||||
logger.info("MQTT connected — subscribing to bms/#")
|
||||
await client.subscribe("bms/#")
|
||||
async for message in client.messages:
|
||||
try:
|
||||
payload = json.loads(message.payload.decode())
|
||||
await process_message(str(message.topic), payload)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing message on {message.topic}: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"MQTT connection failed: {e} — retrying in 5s")
|
||||
await asyncio.sleep(5)
|
||||
234
backend/services/seed.py
Normal file
234
backend/services/seed.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
"""
|
||||
Seed the database with default sensor registry and alarm threshold rules.
|
||||
Runs on startup if tables are empty — subsequent restarts are no-ops.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SITE_ID = "sg-01"
|
||||
|
||||
# ── Threshold seed data ────────────────────────────────────────────────────────
|
||||
# (sensor_type, threshold_value, direction, severity, message_template, locked)
|
||||
# locked=True → state-machine encoding, hidden from UI
|
||||
# locked=False → numeric setpoint, user-editable
|
||||
|
||||
THRESHOLD_SEED_DATA: list[tuple] = [
|
||||
# Rack environment
|
||||
("temperature", 28.0, "above", "warning", "Temperature elevated at {sensor_id}: {value:.1f}°C", False),
|
||||
("temperature", 32.0, "above", "critical", "Temperature critical at {sensor_id}: {value:.1f}°C", False),
|
||||
("humidity", 65.0, "above", "warning", "Humidity elevated at {sensor_id}: {value:.0f}%", False),
|
||||
# PDU / rack power
|
||||
("power_kw", 7.5, "above", "warning", "PDU load elevated at {sensor_id}: {value:.1f} kW", False),
|
||||
("power_kw", 9.5, "above", "critical", "PDU load critical at {sensor_id}: {value:.1f} kW", False),
|
||||
# UPS — numeric setpoints
|
||||
("ups_charge", 80.0, "below", "warning", "UPS battery low at {sensor_id}: {value:.0f}%", False),
|
||||
("ups_charge", 50.0, "below", "critical", "UPS battery critical at {sensor_id}: {value:.0f}%", False),
|
||||
("ups_load", 85.0, "above", "warning", "UPS load high at {sensor_id}: {value:.0f}%", False),
|
||||
("ups_load", 95.0, "above", "critical", "UPS load critical at {sensor_id}: {value:.0f}% — overload", False),
|
||||
("ups_runtime", 15.0, "below", "warning", "UPS runtime low at {sensor_id}: {value:.0f} min remaining", False),
|
||||
("ups_runtime", 5.0, "below", "critical", "UPS runtime critical at {sensor_id}: {value:.0f} min — imminent shutdown", False),
|
||||
# UPS — state transitions (locked)
|
||||
("ups_state", 0.5, "above", "critical", "UPS switched to battery at {sensor_id} — mains power lost", True),
|
||||
("ups_state", 1.5, "above", "critical", "UPS overloaded at {sensor_id} — immediate risk of failure", True),
|
||||
# Leak (locked — binary)
|
||||
("leak", 0.5, "above", "critical", "Water leak detected at {sensor_id}!", True),
|
||||
# CRAC capacity & efficiency
|
||||
("cooling_cap_pct", 90.0, "above", "warning", "CRAC near capacity limit at {sensor_id}: {value:.1f}%", False),
|
||||
("cooling_cop", 1.5, "below", "warning", "CRAC running inefficiently at {sensor_id}: COP {value:.2f}", False),
|
||||
# CRAC compressor
|
||||
("cooling_comp_load", 95.0, "above", "warning", "CRAC compressor overloaded at {sensor_id}: {value:.1f}%", False),
|
||||
("cooling_high_press", 22.0, "above", "critical", "CRAC high refrigerant pressure at {sensor_id}: {value:.1f} bar", False),
|
||||
("cooling_low_press", 3.0, "below", "critical", "CRAC low refrigerant pressure at {sensor_id}: {value:.1f} bar — possible leak", False),
|
||||
("cooling_superheat", 16.0, "above", "warning", "CRAC discharge superheat high at {sensor_id}: {value:.1f}°C", False),
|
||||
# CRAC filter
|
||||
("cooling_filter_dp", 80.0, "above", "warning", "CRAC filter requires attention at {sensor_id}: {value:.0f} Pa", False),
|
||||
("cooling_filter_dp", 120.0, "above", "critical", "CRAC filter critically blocked at {sensor_id}: {value:.0f} Pa — replace now", False),
|
||||
# CRAC return air
|
||||
("cooling_return", 36.0, "above", "warning", "CRAC return air temperature high at {sensor_id}: {value:.1f}°C", False),
|
||||
("cooling_return", 42.0, "above", "critical", "CRAC return air temperature critical at {sensor_id}: {value:.1f}°C", False),
|
||||
# Generator — numeric setpoints
|
||||
("gen_fuel_pct", 25.0, "below", "warning", "Generator fuel low at {sensor_id}: {value:.1f}%", False),
|
||||
("gen_fuel_pct", 10.0, "below", "critical", "Generator fuel critical at {sensor_id}: {value:.1f}%", False),
|
||||
("gen_load_pct", 85.0, "above", "warning", "Generator load high at {sensor_id}: {value:.1f}%", False),
|
||||
("gen_load_pct", 95.0, "above", "critical", "Generator overloaded at {sensor_id}: {value:.1f}%", False),
|
||||
("gen_coolant_c", 95.0, "above", "warning", "Generator coolant temperature high at {sensor_id}: {value:.1f}°C", False),
|
||||
("gen_coolant_c", 105.0, "above", "critical", "Generator coolant critical at {sensor_id}: {value:.1f}°C — risk of shutdown", False),
|
||||
("gen_oil_press", 2.0, "below", "critical", "Generator oil pressure low at {sensor_id}: {value:.1f} bar", False),
|
||||
# Generator — state transitions (locked)
|
||||
("gen_state", 0.5, "above", "warning", "Generator running at {sensor_id} — site is on standby power", True),
|
||||
("gen_state", -0.5, "below", "critical", "Generator fault at {sensor_id} — no standby power available", True),
|
||||
# PDU phase imbalance
|
||||
("pdu_imbalance", 5.0, "above", "warning", "PDU phase imbalance at {sensor_id}: {value:.1f}%", False),
|
||||
("pdu_imbalance", 15.0, "above", "critical", "PDU phase imbalance critical at {sensor_id}: {value:.1f}%", False),
|
||||
# ATS — numeric
|
||||
("ats_ua_v", 50.0, "below", "critical", "Utility A power failure at {sensor_id} — supply lost", False),
|
||||
# ATS — state (locked)
|
||||
("ats_active", 1.5, "above", "warning", "ATS transferred to generator at {sensor_id} — utility power lost", True),
|
||||
# Chiller — numeric
|
||||
("chiller_cop", 2.5, "below", "warning", "Chiller running inefficiently at {sensor_id}: COP {value:.2f}", False),
|
||||
# Chiller — state (locked)
|
||||
("chiller_state", 0.5, "below", "critical", "Chiller fault at {sensor_id} — CHW supply lost", True),
|
||||
# VESDA fire — state (all locked)
|
||||
("vesda_level", 0.5, "above", "warning", "VESDA smoke detected at {sensor_id}: level elevated", True),
|
||||
("vesda_level", 1.5, "above", "warning", "VESDA action threshold reached at {sensor_id}", True),
|
||||
("vesda_level", 2.5, "above", "critical", "VESDA FIRE ALARM at {sensor_id}!", True),
|
||||
("vesda_flow", 0.5, "below", "critical", "VESDA aspirator flow fault at {sensor_id} — detector may be compromised", True),
|
||||
("vesda_det1", 0.5, "below", "warning", "VESDA detector 1 fault at {sensor_id}", True),
|
||||
("vesda_det2", 0.5, "below", "warning", "VESDA detector 2 fault at {sensor_id}", True),
|
||||
# Network — numeric
|
||||
("net_pkt_loss_pct", 1.0, "above", "warning", "Packet loss detected at {sensor_id}: {value:.1f}%", False),
|
||||
("net_pkt_loss_pct", 5.0, "above", "critical", "High packet loss at {sensor_id}: {value:.1f}%", False),
|
||||
("net_temp_c", 65.0, "above", "warning", "Switch temperature high at {sensor_id}: {value:.1f}°C", False),
|
||||
("net_temp_c", 75.0, "above", "critical", "Switch temperature critical at {sensor_id}: {value:.1f}°C", False),
|
||||
# Network — state (locked)
|
||||
("net_state", 0.5, "above", "warning", "Network switch degraded at {sensor_id}", True),
|
||||
("net_state", 1.5, "above", "critical", "Network switch down at {sensor_id} — connectivity lost", True),
|
||||
]
|
||||
|
||||
# ── Sensor seed data ────────────────────────────────────────────────────────────
|
||||
|
||||
def _build_sensor_list() -> list[dict]:
|
||||
sensors = [
|
||||
{"device_id": "gen-01", "name": "Diesel Generator 1", "device_type": "generator", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/generator/gen-01"}},
|
||||
{"device_id": "ups-01", "name": "UPS Unit 1", "device_type": "ups", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/power/ups-01"}},
|
||||
{"device_id": "ups-02", "name": "UPS Unit 2", "device_type": "ups", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/power/ups-02"}},
|
||||
{"device_id": "ats-01", "name": "Transfer Switch 1", "device_type": "ats", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/power/ats/ats-01"}},
|
||||
{"device_id": "chiller-01", "name": "Chiller Plant 1", "device_type": "chiller", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/cooling/chiller/chiller-01"}},
|
||||
{"device_id": "crac-01", "name": "CRAC Unit — Hall A", "device_type": "crac", "room_id": "hall-a", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/cooling/crac-01"}},
|
||||
{"device_id": "crac-02", "name": "CRAC Unit — Hall B", "device_type": "crac", "room_id": "hall-b", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/cooling/crac-02"}},
|
||||
{"device_id": "vesda-hall-a","name": "VESDA Fire Zone — Hall A","device_type": "fire_zone", "room_id": "hall-a", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/fire/vesda-hall-a"}},
|
||||
{"device_id": "vesda-hall-b","name": "VESDA Fire Zone — Hall B","device_type": "fire_zone", "room_id": "hall-b", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/fire/vesda-hall-b"}},
|
||||
{"device_id": "leak-01", "name": "Leak Sensor — CRAC Zone A","device_type": "leak", "room_id": "hall-a", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/leak/leak-01"}},
|
||||
{"device_id": "leak-02", "name": "Leak Sensor — Server Row B1","device_type": "leak", "room_id": "hall-b", "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/leak/leak-02"}},
|
||||
{"device_id": "leak-03", "name": "Leak Sensor — UPS Room", "device_type": "leak", "room_id": None, "rack_id": None, "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/leak/leak-03"}},
|
||||
{"device_id": "sw-core-01", "name": "Core Switch — Hall A", "device_type": "network_switch","room_id": "hall-a", "rack_id": "SG1A01.01", "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/network/sw-core-01"}},
|
||||
{"device_id": "sw-core-02", "name": "Core Switch — Hall B", "device_type": "network_switch","room_id": "hall-b", "rack_id": "SG1B01.01", "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/network/sw-core-02"}},
|
||||
{"device_id": "sw-edge-01", "name": "Edge / Uplink Switch", "device_type": "network_switch","room_id": "hall-a", "rack_id": "SG1A01.05", "protocol": "mqtt", "protocol_config": {"topic": "bms/sg-01/network/sw-edge-01"}},
|
||||
]
|
||||
# Generate racks
|
||||
for room_id, row_prefix in [("hall-a", "SG1A"), ("hall-b", "SG1B")]:
|
||||
for row_num in ["01", "02"]:
|
||||
for rack_num in range(1, 21):
|
||||
rack_id = f"{row_prefix}{row_num}.{rack_num:02d}"
|
||||
sensors.append({
|
||||
"device_id": rack_id,
|
||||
"name": f"Rack PDU — {rack_id}",
|
||||
"device_type": "rack",
|
||||
"room_id": room_id,
|
||||
"rack_id": rack_id,
|
||||
"protocol": "mqtt",
|
||||
"protocol_config": {
|
||||
"env_topic": f"bms/sg-01/{room_id}/{rack_id}/env",
|
||||
"pdu_topic": f"bms/sg-01/{room_id}/{rack_id}/power",
|
||||
},
|
||||
})
|
||||
return sensors
|
||||
|
||||
SENSOR_SEED_DATA = _build_sensor_list()
|
||||
|
||||
# ── Default settings ────────────────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_SETTINGS: dict[str, dict] = {
|
||||
"site": {
|
||||
"name": "Singapore DC01",
|
||||
"timezone": "Asia/Singapore",
|
||||
"description": "Production data centre — Singapore",
|
||||
},
|
||||
"notifications": {
|
||||
"critical_alarms": True,
|
||||
"warning_alarms": True,
|
||||
"generator_events": True,
|
||||
"maintenance_reminders": True,
|
||||
"webhook_url": "",
|
||||
"email_recipients": "",
|
||||
},
|
||||
"integrations": {
|
||||
"mqtt_host": "mqtt",
|
||||
"mqtt_port": 1883,
|
||||
},
|
||||
"page_prefs": {
|
||||
"default_time_range_hours": 6,
|
||||
"refresh_interval_seconds": 30,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Seed functions ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def seed_thresholds(session: AsyncSession) -> None:
|
||||
result = await session.execute(
|
||||
text("SELECT COUNT(*) FROM alarm_thresholds WHERE site_id = :s"),
|
||||
{"s": SITE_ID},
|
||||
)
|
||||
if result.scalar() > 0:
|
||||
return
|
||||
|
||||
for st, tv, direction, severity, msg, locked in THRESHOLD_SEED_DATA:
|
||||
await session.execute(text("""
|
||||
INSERT INTO alarm_thresholds
|
||||
(site_id, sensor_type, threshold_value, direction, severity, message_template, enabled, locked)
|
||||
VALUES
|
||||
(:site_id, :sensor_type, :threshold_value, :direction, :severity, :message_template, true, :locked)
|
||||
"""), {
|
||||
"site_id": SITE_ID, "sensor_type": st, "threshold_value": tv,
|
||||
"direction": direction, "severity": severity,
|
||||
"message_template": msg, "locked": locked,
|
||||
})
|
||||
await session.commit()
|
||||
logger.info(f"Seeded {len(THRESHOLD_SEED_DATA)} alarm threshold rules")
|
||||
|
||||
|
||||
async def seed_sensors(session: AsyncSession) -> None:
|
||||
result = await session.execute(
|
||||
text("SELECT COUNT(*) FROM sensors WHERE site_id = :s"),
|
||||
{"s": SITE_ID},
|
||||
)
|
||||
if result.scalar() > 0:
|
||||
return
|
||||
|
||||
for s in SENSOR_SEED_DATA:
|
||||
await session.execute(text("""
|
||||
INSERT INTO sensors
|
||||
(site_id, device_id, name, device_type, room_id, rack_id, protocol, protocol_config, enabled)
|
||||
VALUES
|
||||
(:site_id, :device_id, :name, :device_type, :room_id, :rack_id, :protocol, :protocol_config, true)
|
||||
ON CONFLICT (site_id, device_id) DO NOTHING
|
||||
"""), {
|
||||
"site_id": SITE_ID,
|
||||
"device_id": s["device_id"],
|
||||
"name": s["name"],
|
||||
"device_type": s["device_type"],
|
||||
"room_id": s.get("room_id"),
|
||||
"rack_id": s.get("rack_id"),
|
||||
"protocol": s["protocol"],
|
||||
"protocol_config": json.dumps(s["protocol_config"]),
|
||||
})
|
||||
await session.commit()
|
||||
logger.info(f"Seeded {len(SENSOR_SEED_DATA)} sensor devices")
|
||||
|
||||
|
||||
async def seed_settings(session: AsyncSession) -> None:
|
||||
for category, defaults in DEFAULT_SETTINGS.items():
|
||||
result = await session.execute(text("""
|
||||
SELECT COUNT(*) FROM site_settings
|
||||
WHERE site_id = :s AND category = :cat AND key = 'config'
|
||||
"""), {"s": SITE_ID, "cat": category})
|
||||
if result.scalar() > 0:
|
||||
continue
|
||||
await session.execute(text("""
|
||||
INSERT INTO site_settings (site_id, category, key, value)
|
||||
VALUES (:site_id, :category, 'config', :value)
|
||||
ON CONFLICT (site_id, category, key) DO NOTHING
|
||||
"""), {"site_id": SITE_ID, "category": category, "value": json.dumps(defaults)})
|
||||
await session.commit()
|
||||
logger.info("Seeded site settings defaults")
|
||||
|
||||
|
||||
async def run_all_seeds(session: AsyncSession) -> None:
|
||||
await seed_thresholds(session)
|
||||
await seed_sensors(session)
|
||||
await seed_settings(session)
|
||||
35
backend/services/ws_manager.py
Normal file
35
backend/services/ws_manager.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import json
|
||||
import logging
|
||||
from fastapi import WebSocket
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self) -> None:
|
||||
self._connections: set[WebSocket] = set()
|
||||
|
||||
async def connect(self, ws: WebSocket) -> None:
|
||||
await ws.accept()
|
||||
self._connections.add(ws)
|
||||
logger.info(f"WS client connected. Total: {len(self._connections)}")
|
||||
|
||||
def disconnect(self, ws: WebSocket) -> None:
|
||||
self._connections.discard(ws)
|
||||
logger.info(f"WS client disconnected. Total: {len(self._connections)}")
|
||||
|
||||
async def broadcast(self, data: dict) -> None:
|
||||
if not self._connections:
|
||||
return
|
||||
message = json.dumps(data, default=str)
|
||||
dead: set[WebSocket] = set()
|
||||
for ws in self._connections:
|
||||
try:
|
||||
await ws.send_text(message)
|
||||
except Exception:
|
||||
dead.add(ws)
|
||||
self._connections -= dead
|
||||
|
||||
|
||||
# Singleton — imported by both the MQTT subscriber and the WS route
|
||||
manager = ConnectionManager()
|
||||
Loading…
Add table
Add a link
Reference in a new issue