diff --git a/README.md b/README.md index 0eeb526..92e2fb9 100644 --- a/README.md +++ b/README.md @@ -43,14 +43,12 @@ cd bms ### 2. Create the environment files -Copy the example files and fill in your values: - ```bash cp backend/.env.example backend/.env cp frontend/.env.local.example frontend/.env.local ``` -Open each file and follow the inline comments. At minimum you need to set the Clerk keys (see [Environment Variables](#environment-variables) below). +The defaults work as-is — they already match the service names in `docker-compose.yml`. **The app runs in demo mode with authentication disabled**, so there are no keys to obtain before the first run. See [Environment Variables](#environment-variables) below. ### 3. Start all services @@ -72,6 +70,14 @@ Open your browser at **http://your-server:5646** ## Environment Variables +> **Authentication is disabled.** The app runs in demo mode: `frontend/proxy.ts` passes every +> request through and there is no auth provider in `app/layout.tsx`. Deploy it on a trusted +> network (LAN, VPN, or behind an authenticating reverse proxy) — every route is open. +> +> To turn auth back on: reinstall `@clerk/nextjs`, wrap the tree in `` in +> `app/layout.tsx`, restore `` in `components/layout/topbar.tsx`, re-add the +> `sign-in` / `sign-up` route groups, and swap `proxy.ts` for `clerkMiddleware`. + ### `backend/.env` ```env @@ -82,11 +88,6 @@ DATABASE_URL=postgresql+asyncpg://dcim:dcim_pass@db:5432/dcim MQTT_HOST=mqtt MQTT_PORT=1883 -# Clerk authentication -# Get these from https://dashboard.clerk.com → Your App → API Keys -CLERK_SECRET_KEY=sk_test_REPLACE_ME -CLERK_JWKS_URL=https://YOUR_APP.clerk.accounts.dev/.well-known/jwks.json - # CORS — add your frontend origin if you expose the backend directly # Leave empty when using the built-in Next.js proxy (recommended) CORS_ORIGINS=[] @@ -97,23 +98,13 @@ DEBUG=true ### `frontend/.env.local` ```env -# Clerk authentication (same app as above) -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_REPLACE_ME -CLERK_SECRET_KEY=sk_test_REPLACE_ME - -# Clerk redirect paths — no need to change these -NEXT_PUBLIC_CLERK_SIGN_IN_URL=/sign-in -NEXT_PUBLIC_CLERK_SIGN_UP_URL=/sign-up -NEXT_PUBLIC_CLERK_AFTER_SIGN_IN_URL=/dashboard -NEXT_PUBLIC_CLERK_AFTER_SIGN_UP_URL=/dashboard - # API path — leave as-is, Next.js proxies /api/backend/* to the backend internally NEXT_PUBLIC_API_URL=/api/backend -``` -> **Where do I get Clerk keys?** -> Sign up free at https://clerk.com → create an application → go to **API Keys**. -> Copy the **Publishable key** and **Secret key** into both files above. +# Backend internal URL (used by the Next.js server-side proxy, not sent to the browser) +# In Docker: http://backend:8000 In local dev: http://localhost:8000 +BACKEND_INTERNAL_URL=http://backend:8000 +``` --- @@ -181,6 +172,26 @@ Browser → Reverse Proxy → :5646 (Next.js) --- +## Deploying to the Unraid Server + +The live instance runs on the Unraid Docker host at `192.168.1.249` as the Portainer stack +**`bms`** — reachable at **http://192.168.1.249:5646/dashboard**. + +```bash +python3 ops/deploy.py # build all four images + deploy/update the stack +python3 ops/deploy.py --no-build # redeploy the stack using the existing images +``` + +There is no `docker` CLI in WSL and no SSH to the Unraid box, so `ops/deploy.py` ships each +service's build context to the remote daemon's `/build` endpoint over the Portainer API +(token at `~/.portainer-token`), then pushes `docker-compose.prod.yml` as a Portainer stack. +The script is idempotent — rerun it after any code change. + +`docker-compose.prod.yml` differs from the dev `docker-compose.yml`: images instead of build +contexts, no source bind mounts, no `--reload`, and only the frontend publishes a host port. +Postgres data is bind-mounted to `/mnt/user/appdata/bms/db` so it lands on the Unraid array +instead of growing inside `docker.img`, and it survives redeploys. + ## Local Development (without Docker) Useful if you want hot-reload on the frontend or backend without rebuilding containers. diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..2429a30 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,116 @@ +# Production stack — deployed to the Unraid Docker host as a Portainer stack. +# +# Differences from docker-compose.yml (the local dev file): +# * No build: sections. ops/deploy.py builds the four images on the Unraid daemon +# first; this file only references them by tag. +# * No source bind mounts and no uvicorn --reload. Code lives in the image. +# * No env_file. Everything is set inline — there are no .env files on the host. +# * Only the frontend publishes a host port. Postgres and MQTT stay on the +# internal bms_net network, as the README's reverse-proxy note recommends. +# * Postgres data is bind-mounted to /mnt/user/appdata/bms/db so it lands on the +# Unraid array rather than growing inside docker.img. +# +# Deploy / redeploy with: python3 ops/deploy.py + +services: + + # ── MQTT Broker ────────────────────────────────────────────────── + mqtt: + image: bms-mqtt:latest + container_name: bms_mqtt + restart: unless-stopped + networks: [bms_net] + healthcheck: + test: ["CMD-SHELL", "mosquitto_sub -t '$$SYS/#' -C 1 -i healthcheck -W 3"] + interval: 10s + timeout: 5s + retries: 5 + + # ── PostgreSQL + TimescaleDB ───────────────────────────────────── + db: + image: timescale/timescaledb:latest-pg16 + container_name: bms_db + restart: unless-stopped + networks: [bms_net] + environment: + POSTGRES_USER: dcim + POSTGRES_PASSWORD: dcim_pass + POSTGRES_DB: dcim + volumes: + - /mnt/user/appdata/bms/db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dcim -d dcim"] + interval: 10s + timeout: 5s + retries: 5 + + # ── FastAPI backend ────────────────────────────────────────────── + backend: + image: bms-backend:latest + container_name: bms_backend + restart: unless-stopped + networks: [bms_net] + environment: + DATABASE_URL: postgresql+asyncpg://dcim:dcim_pass@db:5432/dcim + MQTT_HOST: mqtt + MQTT_PORT: "1883" + CORS_ORIGINS: "[]" + DEBUG: "false" + depends_on: + db: + condition: service_healthy + mqtt: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8000/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + + # ── Simulator bots (seed first, then run bots) ─────────────────── + simulators: + image: bms-simulators:latest + container_name: bms_simulators + restart: unless-stopped + networks: [bms_net] + environment: + MQTT_HOST: mqtt + MQTT_PORT: "1883" + DATABASE_URL: postgresql://dcim:dcim_pass@db:5432/dcim + SEED_MINUTES: "30" + depends_on: + db: + condition: service_healthy + mqtt: + condition: service_healthy + backend: + condition: service_healthy + + # ── Next.js frontend (the only publicly reachable service) ─────── + frontend: + image: bms-frontend:latest + container_name: bms_frontend + restart: unless-stopped + networks: [bms_net] + ports: + - "5646:5646" + environment: + PORT: "5646" + HOSTNAME: "0.0.0.0" + NODE_ENV: production + BACKEND_INTERNAL_URL: http://backend:8000 + depends_on: + - backend + +networks: + # The Unraid daemon's default address pools are fully subnetted — 24 existing + # stacks have claimed every 172.17-172.31/16 and 192.168.x/20 slot, so letting + # Docker auto-allocate fails with "all predefined address pools have been fully + # subnetted". An explicit subnet bypasses the allocator entirely. + # 172.31.42.0/24 is free: the only other 172.31 tenant is wg0 on 172.31.200.0/24. + bms_net: + driver: bridge + ipam: + config: + - subnet: 172.31.42.0/24 diff --git a/infra/mosquitto/Dockerfile b/infra/mosquitto/Dockerfile new file mode 100644 index 0000000..1ee8a31 --- /dev/null +++ b/infra/mosquitto/Dockerfile @@ -0,0 +1,11 @@ +# Mosquitto with the broker config baked in. +# +# The compose file bind-mounts ./infra/mosquitto/mosquitto.conf, which only works +# when the repo is checked out on the Docker host. The Unraid deploy builds images +# remotely via the Portainer Docker API and has no repo on the host, so the config +# travels inside the image instead. +FROM eclipse-mosquitto:2 + +COPY mosquitto.conf /mosquitto/config/mosquitto.conf + +EXPOSE 1883 diff --git a/memory.md b/memory.md new file mode 100644 index 0000000..fc0256a --- /dev/null +++ b/memory.md @@ -0,0 +1,85 @@ +# BMS — memory + +DCIM demo platform: Next.js 16 frontend, FastAPI backend, TimescaleDB, Mosquitto MQTT, +and Python simulator bots that publish fake data-centre telemetry. + +Live at **http://192.168.1.249:5646/dashboard** (Portainer stack `bms` on Unraid). + +## Decisions & rationale + +- **Auth is disabled (demo mode).** The repo shipped half-converted: `proxy.ts` already said + "auth is disabled for demo mode", but `app/layout.tsx` still wrapped the tree in + `` and `topbar.tsx` used ``. `@clerk/nextjs` v7 fails `next build` + without a real publishable key, so the frontend image could not build at all. Finished the + conversion instead of buying Clerk keys: removed `ClerkProvider`/`UserButton`, deleted the + `sign-in`/`sign-up` route groups, and dropped `@clerk/nextjs` from `package.json`. The service + is LAN-only and reachable remotely over Tailscale, so the tailnet is the auth boundary. +- **Deploy via the Portainer API, not compose-on-host.** No `docker` CLI in WSL and Unraid's + SSH (port 22) is closed, so `ops/deploy.py` tars each service's build context, POSTs it to the + remote daemon's `/build` endpoint, then pushes `docker-compose.prod.yml` as a Portainer stack. + Same pattern as `toknmtr/ops/deploy.py`. +- **Separate `docker-compose.prod.yml`.** The dev compose file bind-mounts source into the + containers and runs `uvicorn --reload` — wrong for a long-running deploy, and impossible + anyway since the repo is not checked out on the Unraid host. The prod file references + prebuilt image tags, sets env inline (no `.env` files on the host), and publishes only the + frontend's port. +- **Mosquitto config is baked into an image** (`infra/mosquitto/Dockerfile`) rather than + bind-mounted, for the same reason: no repo on the host. +- **`bms_net` pins subnet 172.31.42.0/24.** Docker's default address pools on that host are + fully subnetted — 24 existing stacks have claimed every `172.17-172.31/16` and + `192.168.x/20` slot, so auto-allocation fails with "all predefined address pools have been + fully subnetted". An explicit subnet bypasses the allocator without disturbing other stacks. + Only other 172.31 tenant is `wg0` on 172.31.200.0/24. +- **Postgres data bind-mounts to `/mnt/user/appdata/bms/db`**, not a named volume, so a + growing timeseries DB lands on the array instead of filling Unraid's fixed-size `docker.img`. +- **`pnpm-workspace.yaml` must be copied in the frontend Dockerfile's deps stage.** It carries + the build-script allowlist; without it pnpm aborts with `ERR_PNPM_IGNORED_BUILDS`. The file + now declares both `allowBuilds` (pnpm 11 spelling) and `ignoredBuiltDependencies` (pnpm 10). + `sharp`, `unrs-resolver` and `msw` are all declined — the app has no `next/image`, so sharp + is dead weight. `packageManager: pnpm@11.20.0` is pinned so a future pnpm release cannot + silently change this again. + +## Open questions / TODOs + +- [ ] Postgres uses the repo's default credentials (`dcim`/`dcim_pass`). Fine while the DB has + no published host port and the box is LAN-only, but change both the compose env and + `DATABASE_URL` together if the port is ever exposed. +- [ ] The simulator floods logs with `mqtt WARNING There are N pending publish calls.` — it + publishes faster than aiomqtt drains. Cosmetic, but it makes `docker logs` near-useless + and grows the log file. Worth batching or throttling the bots. +- [ ] `backend/api/routes/ws.py` exists but nothing in the frontend opens a WebSocket (the UI + polls every 15 s). Either wire it up or drop it. Note: Next.js rewrites do not proxy WS + upgrades, so a WS client would need to reach the backend directly. +- [ ] No host port is published for Postgres or MQTT. Add a `ports:` entry to + `docker-compose.prod.yml` if direct DB access or external MQTT publishing is ever needed. +- [ ] No reverse-proxy entry yet. Add an Nginx Proxy Manager host pointing at `:5646` if this + should be reachable by name rather than IP:port. + +## Session log + +### 2026-08-04 + +- Cloned the Forgejo repo into `~/claude/projects/BMS` and deployed it to the Unraid host. +- Finished the demo-mode auth conversion (see Decisions) so the frontend image could build; + regenerated `pnpm-lock.yaml` without `@clerk/nextjs`. Verified `next build` locally — all + 18 routes prerender. +- Hit and fixed three deploy blockers in order: (1) the frontend Dockerfile never copied + `pnpm-workspace.yaml`, so pnpm's ignored-builds allowlist was invisible inside the build; + (2) pnpm 11 renamed that setting to `allowBuilds` and ignores `ignoredBuiltDependencies`; + (3) the Unraid daemon's Docker address pools were exhausted, so the stack network could not + be created. +- Backend's host port 8000 deliberately not published — Portainer's Edge tunnel already owns + 8000 on that host. Ports 1883/5433/5646 were free; only 5646 is published. +- Added `ops/deploy.py`, `docker-compose.prod.yml`, `infra/mosquitto/Dockerfile`; updated the + README (demo-mode note, Unraid deploy section) and both `.env` examples. +- Verified live: all 5 containers healthy, 30 min of seeded history present, telemetry + flowing, alarm engine firing (55 active), dashboard screenshotted and rendering correctly. + +## External references + +- Repo: https://git.rdx4.com/megaproxy/BMS +- Live dashboard: http://192.168.1.249:5646/dashboard +- API docs (Swagger): http://192.168.1.249:5646/api/backend/docs +- Portainer (stack `bms`, endpoint id 3): http://192.168.1.249:9000 — token at `~/.portainer-token` +- Postgres data on the host: `/mnt/user/appdata/bms/db` +- Deploy pattern copied from: `~/claude/projects/toknmtr/ops/deploy.py` diff --git a/ops/deploy.py b/ops/deploy.py new file mode 100644 index 0000000..9d874fb --- /dev/null +++ b/ops/deploy.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Deploy the BMS stack to the Unraid Docker host via the Portainer API. + +Why this shape: there is no usable `docker` CLI in WSL, no SSH to the Unraid box +(port 22 is closed) and no shared registry the Unraid daemon can pull from. So each +service's build *context* is packed into a tar and POSTed to the remote daemon's +`/build` endpoint — Unraid runs the Dockerfiles itself. Once the four images exist +locally on that daemon, docker-compose.prod.yml is pushed as a Portainer stack, +which references the images by tag and never needs the repo on the host. + +Mirrors the pattern in ~/claude/projects/toknmtr/ops/deploy.py. +Idempotent: re-running rebuilds every image and updates the existing stack in place. +The Postgres bind mount at /mnt/user/appdata/bms/db survives redeploys. + +Usage: + python3 ops/deploy.py # build everything + deploy the stack + python3 ops/deploy.py --no-build # redeploy the stack using existing images +""" +import io +import json +import os +import sys +import tarfile +import time +import urllib.error +import urllib.request + +PORTAINER = "http://192.168.1.249:9000" +EP = 3 # 'local' docker endpoint (unix:///var/run/docker.sock) +DOCKER = f"{PORTAINER}/api/endpoints/{EP}/docker" + +HOST = "192.168.1.249" +FRONTEND_PORT = "5646" +STACK_NAME = "bms" +APPDATA = "/mnt/user/appdata/bms" + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +COMPOSE_FILE = os.path.join(PROJECT_DIR, "docker-compose.prod.yml") + +# (image tag, context subdirectory) — built in this order. +# mqtt/backend first so the slow frontend build is last and failures surface early. +IMAGES = [ + ("bms-mqtt:latest", "infra/mosquitto"), + ("bms-backend:latest", "backend"), + ("bms-simulators:latest", "simulators"), + ("bms-frontend:latest", "frontend"), +] + +# Never shipped in a build context. +EXCLUDE_DIRS = {".git", "node_modules", ".next", "__pycache__", ".venv", "venv", + ".vscode", ".idea", ".claude"} +EXCLUDE_SUFFIXES = (".pyc", ".pyo", ".log", ".tsbuildinfo") +EXCLUDE_FILES = {".env", ".env.local", ".DS_Store", "Thumbs.db"} + +TOKEN = open(os.path.expanduser("~/.portainer-token")).read().strip() +HDR = {"X-API-Key": TOKEN} + + +def req(url, method, data=None, headers=None, raw=False, timeout=900): + h = dict(HDR) + if headers: + h.update(headers) + body = data + if data is not None and not raw: + body = json.dumps(data).encode() + h["Content-Type"] = "application/json" + r = urllib.request.Request(url, data=body, headers=h, method=method) + try: + with urllib.request.urlopen(r, timeout=timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def docker(method, path, **kw): + return req(DOCKER + path, method, **kw) + + +def api(method, path, **kw): + return req(PORTAINER + "/api" + path, method, **kw) + + +# ── image builds ───────────────────────────────────────────────────── + +def context_tar(subdir): + """Pack one service directory (minus excludes) for the Docker /build API.""" + root_dir = os.path.join(PROJECT_DIR, subdir) + if not os.path.isdir(root_dir): + sys.exit(f"ERROR: build context {subdir} does not exist.") + if not os.path.exists(os.path.join(root_dir, "Dockerfile")): + sys.exit(f"ERROR: no Dockerfile in {subdir}.") + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for root, dirs, files in os.walk(root_dir): + dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS] + for fn in files: + if fn in EXCLUDE_FILES or fn.endswith(EXCLUDE_SUFFIXES): + continue + full = os.path.join(root, fn) + tar.add(full, arcname=os.path.relpath(full, root_dir)) + return buf.getvalue() + + +def build_image(tag, subdir): + ctx = context_tar(subdir) + print(f"\n── building {tag} from {subdir}/ ({len(ctx)/1024:.0f} KiB context)") + status, out = docker( + "POST", + f"/build?t={tag}&dockerfile=Dockerfile&forcerm=true", + data=ctx, + headers={"Content-Type": "application/x-tar"}, + raw=True, + timeout=1800, # the Next.js build is the slow one + ) + text = out.decode(errors="replace") + errored = None + last = "" + for line in text.splitlines(): + try: + msg = json.loads(line) + except ValueError: + continue + if "stream" in msg: + s = msg["stream"].rstrip() + if s: + last = s + if s.startswith("Step ") or "Successfully" in s: + print(" " + s) + if "error" in msg: + errored = msg["error"] + if status != 200 or errored: + print(f"BUILD FAILED ({tag}) — status {status}") + print(text[-3000:]) + sys.exit(f"ERROR: build of {tag} failed.") + print(f" OK: {last[:120]}") + + +# ── host prep ──────────────────────────────────────────────────────── + +def ensure_appdata(): + """Create /mnt/user/appdata/bms/db before Postgres bind-mounts it. + + dockerd would auto-create the path, but doing it explicitly turns a silent + permission problem into a visible one. + """ + print(f"\n── ensuring {APPDATA}/db exists on the host") + docker("POST", "/images/create?fromImage=busybox&tag=latest", data=b"", raw=True, + headers={"Content-Type": "application/json"}, timeout=300) + body = { + "Image": "busybox:latest", + "Cmd": ["mkdir", "-p", "/hostdata/bms/db"], + "HostConfig": {"Binds": ["/mnt/user/appdata:/hostdata"], "AutoRemove": True}, + } + status, out = docker("POST", "/containers/create?name=bms_mkdir_oneshot", data=body) + if status not in (200, 201): + print(" warn: could not create helper container:", + out.decode(errors="replace")[:200]) + print(" continuing — dockerd will create the bind path itself") + return + cid = json.loads(out)["Id"] + docker("POST", f"/containers/{cid}/start") + docker("POST", f"/containers/{cid}/wait", data={}, timeout=120) + print(" OK") + + +# ── stack deploy ───────────────────────────────────────────────────── + +def find_stack(): + status, out = api("GET", "/stacks") + if status != 200: + sys.exit(f"ERROR: could not list stacks ({status}): " + f"{out.decode(errors='replace')[:300]}") + for s in json.loads(out): + if s.get("Name") == STACK_NAME: + return s + return None + + +def deploy_stack(): + compose = open(COMPOSE_FILE).read() + existing = find_stack() + + if existing: + sid = existing["Id"] + print(f"\n── updating existing stack '{STACK_NAME}' (id {sid})") + status, out = api( + "PUT", f"/stacks/{sid}?endpointId={EP}", + data={"stackFileContent": compose, "env": existing.get("Env", []), + "prune": True, "pullImage": False}, + timeout=900, + ) + else: + print(f"\n── creating stack '{STACK_NAME}'") + status, out = api( + "POST", f"/stacks/create/standalone/string?endpointId={EP}", + data={"name": STACK_NAME, "stackFileContent": compose, "env": []}, + timeout=900, + ) + if status == 404: + # Portainer < 2.19 route + status, out = api( + "POST", f"/stacks?type=2&method=string&endpointId={EP}", + data={"Name": STACK_NAME, "StackFileContent": compose, "Env": []}, + timeout=900, + ) + + if status not in (200, 201): + print("STACK DEPLOY FAILED — status", status) + print(out.decode(errors="replace")[:3000]) + sys.exit("ERROR: stack deploy failed.") + print(" stack deployed") + + +# ── verify ─────────────────────────────────────────────────────────── + +def container_states(): + status, out = docker("GET", "/containers/json?all=1") + if status != 200: + return {} + return {n.lstrip("/"): (c.get("State"), c.get("Status")) + for c in json.loads(out) for n in c.get("Names", []) + if n.lstrip("/").startswith("bms_")} + + +def verify(): + print("\n── verifying") + home = f"http://{HOST}:{FRONTEND_PORT}/dashboard" + health = f"http://{HOST}:{FRONTEND_PORT}/api/backend/api/health" + ok = False + for i in range(40): + time.sleep(5) + try: + with urllib.request.urlopen(health, timeout=10) as r: + j = json.load(r) + with urllib.request.urlopen(home, timeout=15) as r2: + code = r2.status + print(f" OK after {5*(i+1)}s — backend health={j}, /dashboard HTTP {code}") + ok = True + break + except Exception as e: + print(f" ...not ready ({type(e).__name__}: {str(e)[:70]})") + + print("\n container states:") + for name, (state, st) in sorted(container_states().items()): + print(f" {name:20s} {state:10s} {st}") + + if not ok: + sys.exit("ERROR: stack did not come up healthy in time.") + + +def main(): + if "--no-build" not in sys.argv: + for tag, subdir in IMAGES: + build_image(tag, subdir) + else: + print("Skipping image builds (--no-build).") + ensure_appdata() + deploy_stack() + verify() + print(f"\nDeployed. Dashboard: http://{HOST}:{FRONTEND_PORT}/dashboard") + print(f"API docs: http://{HOST}:{FRONTEND_PORT}/api/backend/docs") + + +if __name__ == "__main__": + main()