fly.toml

# fly.toml app configuration file generated for SourceBox Sentry Command
#
# See https://fly.io/docs/reference/configuration/ for information about settings

app = "sentinel-command"
primary_region = "sjc"

[build]

[env]
  FRONTEND_URL = "https://app.sentinel-command.com"
  # DATABASE_URL is deliberately NOT here any more. It points at Postgres
  # now, which makes it a credential, so it lives in `fly secrets` —
  # `fly.toml` is committed to a public repository.
  #
  # Self-hosted installs are unaffected: they get the SQLite default from
  # app/core/config.py and the data layer branches on the URL scheme.
  #
  # Segment-cache ceiling MUST stay well under [[vm]].memory_mb below —
  # above physical RAM, the kernel OOM-killer fires (killing every
  # org's streams at once) long before the cache's own eviction can.
  # 384 MiB on the 1 GiB machine leaves ~600 MiB for Python
  # (~25 concurrent live cameras). Scale this and memory_mb TOGETHER.
  # The database no longer shares this machine's RAM for page cache,
  # so there is a little more headroom here than there used to be.
  SEGMENT_CACHE_MAX_TOTAL_BYTES = "402653184"

# This app runs TWO process groups from ONE image. Fly has no per-group
# image, only per-group command — which is why the Sentinel AI agent
# resolves against backend/pyproject.toml rather than carrying its own
# dependency set.
#
#   app   — the web tier. Owns the volume and the HLS segment cache,
#           always-on (min_machines_running = 1 below).
#   agent — the Sentinel AI worker. No volume, ALSO always-on, woken by
#           the HMAC-signed wakeup webhook over .flycast.
#
#           This said "scales to zero" until 2026-09-14 and had been wrong
#           since the LiteLLM move: the service block below sets
#           auto_stop_machines = "off" with min_machines_running = 1, and
#           the machine has been `started` continuously. The reasoning for
#           keeping it warm — Fly's proxy gives an auto-started machine
#           only ~8s to bind, and this process needed ~10s — is argued at
#           length beside that block. Scaling to zero is the shape this
#           worker *wants*; it is not the shape it has.
#
# The agent is a separate PROCESS GROUP rather than a thread inside the
# web app on purpose: a run holds base64 frames for up to 270s, and the
# segment cache is already budgeted at 384 MiB of this machine's 1 GiB
# (see SEGMENT_CACHE_MAX_TOTAL_BYTES above). Sharing one machine is how
# the OOM killer ends up taking every org's streams down at once.
#
# Defining [processes] overrides the Dockerfile CMD for BOTH groups, so
# the web command below must stay in sync with that CMD.
[processes]
  app = "/app/.venv/bin/uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 1 --timeout-keep-alive 65 --forwarded-allow-ips=* --no-access-log"
  agent = "/app/.venv/bin/python -m app.sentinel_agent"

# Still required after the Postgres migration — /data holds HLS segment
# working files and the local backup directory. It is no longer where
# the database lives, which also means health_probes' disk check now
# measures segment storage rather than database growth.
#
# `processes` is REQUIRED here now. Without it the mount applies to every
# process group, and the agent machine fails to boot fighting the web
# machine for the volume's single attachment slot.
[[mounts]]
  source = "sentinel_data"
  destination = "/data"
  processes = ["app"]

[deploy]
  # CI uses build-image-then-`fly machine update` (see
  # .github/workflows/deploy.yml) which doesn't honour this strategy
  # — it goes straight to the in-place machine API.  This block
  # only matters if someone runs plain `fly deploy` manually.
  #
  # `immediate` was forced by SQLite-on-a-single-volume: rolling tries
  # to provision a parallel machine and errored on the volume's single
  # attachment slot. With the database on Postgres that constraint is
  # gone, so this now uses the default rolling strategy.
  #
  # Note the volume above still pins us to one machine for HLS segment
  # state, so a deploy is still a brief restart. Genuine zero-downtime
  # needs a second machine — which is now *possible* where it wasn't
  # before, rather than something this line alone delivers.

[http_service]
  internal_port = 8000
  force_https = true
  auto_stop_machines = "off"
  auto_start_machines = true
  min_machines_running = 1
  processes = ["app"]

  [[http_service.checks]]
    grace_period = "30s"
    interval = "10s"
    timeout = "5s"
    method = "GET"
    path = "/api/health"

# Service for the agent process group. Command Center reaches it at
# http://sentinel-command.flycast:8080 over 6PN, which is where
# SENTINEL_AGENT_WEBHOOK_URL points, and that requires a private IPv6 on
# the app: `fly ips allocate-v6 --private`.
#
# THIS IS ALSO REACHABLE FROM THE PUBLIC INTERNET, in cleartext. An
# earlier version of this comment claimed "not exposed publicly"; that
# was wrong, and verified wrong on 2026-09-12:
#
#   $ curl http://sentinel-command.fly.dev:8080/health
#   HTTP 200  {"status":"ok"}        # no TLS available on this port
#
# Fly's rule is that "if you have public IP addresses assigned to your
# app, then services in fly.toml are exposed to the public internet" —
# and this app needs public IPs for the web tier. The two process groups
# share one app, so the agent's service cannot be made private while the
# web service is public.
#
# The obvious fix does NOT work: at least one [[services.ports]] entry is
# required per [[services]] section, and without one the service is
# unreachable over Flycast too, which would break every wakeup. Making
# this private-only means moving the agent to its own app with no public
# IPs — a real change, not a config tweak.
#
# What actually protects it today is the authentication, which was
# probed on the live service: POST /wakeup rejects a bad HMAC with 401,
# bodies carry a timestamp checked against a ±5 min window so captured
# requests cannot be replayed, and the unauthenticated dev trigger on
# POST / is disabled whenever WEBHOOK_VERIFY_SIGNATURE is set (it
# defaults to True, which is what production runs).
#
# NOT scaled to zero, despite that being this worker's obvious shape, and
# despite it being how the agent ran as its own app. Fly's proxy waits
# only ~8s for a machine it auto-started to bind its port, and this
# process cannot reliably beat that: Python plus the MCP SDK plus Sentry
# took ~7s before LiteLLM and ~10s after — so an auto-started machine got
# declared unreachable and Command Center's wakeup came back
# RemoteDisconnected. Measured, twice, on 2026-09-09.
#
# The 7s figure means this was always marginal and happened to fit; the
# LiteLLM move only exposed it. Deferring litellm's import cut boot from
# 16s to 10s, which was necessary but not sufficient, and trimming
# further imports would be tuning against a proxy limit we do not
# control and cannot test except in production.
#
# So the agent stays warm. One shared-cpu-1x/512MB machine is ~$2/month,
# which is noise against the LLM spend of a single run, and it removes
# cold starts from the wakeup path entirely rather than racing them.
[[services]]
  internal_port = 8080
  protocol = "tcp"
  auto_stop_machines = "off"
  auto_start_machines = true
  min_machines_running = 1
  processes = ["agent"]

  [[services.ports]]
    port = 8080

  # The agent ran with NO health check at all until 2026-09-12, which made
  # it the least observed machine in the system despite being the one
  # deliberately kept running 24/7. `fly machines list` showed CHECKS
  # empty for this group while the app group showed 1/1. The entry point's
  # own docstring already claimed "/health is what Fly's checks and any
  # external monitor hit" — it wasn't.
  #
  # Note what this does and does not buy. A failing check removes the
  # machine from the load balancer; it does NOT restart it (that is the
  # [[restart]] policy's job). So this makes a wedged agent *visible* and
  # makes wakeups fail fast rather than hang on a process that accepts TCP
  # and does nothing. It is not self-healing.
  #
  # Safe to run against this process because the agent is async all the
  # way down — the LLM call is `await litellm.acompletion(...)` and the MCP
  # client is an async session, so nothing blocks the event loop during an
  # investigation and /health keeps answering while a run is in flight. A
  # blocking LLM call here would have made this check *cause* wakeup
  # failures during the agent's most important work.
  #
  # grace_period covers cold boot, measured at ~10s once litellm's import
  # was deferred, with headroom.
  [[services.http_checks]]
    grace_period = "30s"
    interval = "15s"
    timeout = "5s"
    method = "get"
    path = "/health"
    protocol = "http"

[[vm]]
  cpu_kind = "shared"
  cpus = 1
  memory_mb = 1024
  processes = ["app"]

# The agent needs less than the web tier: no segment cache, no SPA, one
# run at a time. 512 MiB comfortably holds the Python process plus the
# base64 frames of a single investigation.
#
# It bills continuously, not per run — this machine is the one kept warm
# on purpose (auto_stop_machines = "off", min_machines_running = 1, see
# the service block above). An earlier version of this comment said it
# "only bills while a run is actually in flight", which contradicted the
# always-warm decision argued twenty lines above it in this same file.
# The ~$2/month figure in that argument is the real number.
[[vm]]
  cpu_kind = "shared"
  cpus = 1
  memory_mb = 512
  processes = ["agent"]