Self-host backend on PostgreSQL alone (Backend:Mode=Postgres, D54)

db/migrations/0003_selfhost_runtime.sql

@@ -0,0 +1,112 @@
+-- Runtime tables for the self-host backend (Backend:Mode=Postgres, D53): queues, delivery state, rollups, event
+-- archive, idempotency keys and monthly counters. The AWS backend uses SQS, DynamoDB and S3 instead and ignores these.
+
+-- A durable queue: messages become visible at visible_at, are leased by pushing visible_at forward, and move to
+-- "<queue>-dlq" after too many receives (the same contract as SQS standard queues).
+CREATE TABLE webhook.queue_messages (
+    id            bigserial PRIMARY KEY,
+    queue         text        NOT NULL,
+    body          text        NOT NULL,
+    enqueued_at   timestamptz NOT NULL DEFAULT now(),
+    visible_at    timestamptz NOT NULL DEFAULT now(),
+    receive_count integer     NOT NULL DEFAULT 0,
+    receipt       uuid
+);
+CREATE INDEX queue_messages_ready_idx ON webhook.queue_messages (queue, visible_at, id);
+
+-- Non-success deliveries only (docs/data-plane.md section 4).
+CREATE TABLE webhook.delivery_state (
+    tenant_id        text        NOT NULL,
+    delivery_id      text        NOT NULL,
+    endpoint_id      text        NOT NULL,
+    event_id         text        NOT NULL,
+    event_type       text        NOT NULL,
+    occurred_at      timestamptz NOT NULL,
+    status           text        NOT NULL,
+    attempt          integer     NOT NULL,
+    tier             text        NOT NULL,
+    next_attempt_at  timestamptz,
+    last_status_code integer,
+    last_error       text,
+    payload          text,
+    payload_ref      text,
+    created_at       timestamptz NOT NULL,
+    updated_at       timestamptz NOT NULL,
+    expires_at       timestamptz NOT NULL,
+    PRIMARY KEY (tenant_id, delivery_id)
+);
+CREATE INDEX delivery_state_due_idx ON webhook.delivery_state (next_attempt_at) WHERE status = 'Scheduled';
+CREATE INDEX delivery_state_tenant_status_idx ON webhook.delivery_state (tenant_id, status, updated_at DESC);
+CREATE INDEX delivery_state_endpoint_idx ON webhook.delivery_state (tenant_id, endpoint_id, updated_at DESC);
+CREATE INDEX delivery_state_expiry_idx ON webhook.delivery_state (expires_at);
+
+-- Per tenant/endpoint and minute counters and latency histograms, added to with ON CONFLICT (docs/data-plane.md).
+CREATE TABLE webhook.rollups (
+    pk          text        NOT NULL,
+    sk          text        NOT NULL,
+    expires_at  timestamptz NOT NULL,
+    created bigint NOT NULL DEFAULT 0,
+    attempts bigint NOT NULL DEFAULT 0,
+    succeeded bigint NOT NULL DEFAULT 0,
+    retried bigint NOT NULL DEFAULT 0,
+    failed bigint NOT NULL DEFAULT 0,
+    exhausted bigint NOT NULL DEFAULT 0,
+    timeouts bigint NOT NULL DEFAULT 0,
+    connection_ms bigint NOT NULL DEFAULT 0,
+    rt_sum bigint NOT NULL DEFAULT 0,
+    qw_sum bigint NOT NULL DEFAULT 0,
+    rt0 bigint NOT NULL DEFAULT 0,
+    rt1 bigint NOT NULL DEFAULT 0,
+    rt2 bigint NOT NULL DEFAULT 0,
+    rt3 bigint NOT NULL DEFAULT 0,
+    rt4 bigint NOT NULL DEFAULT 0,
+    rt5 bigint NOT NULL DEFAULT 0,
+    rt6 bigint NOT NULL DEFAULT 0,
+    rt7 bigint NOT NULL DEFAULT 0,
+    rt8 bigint NOT NULL DEFAULT 0,
+    rt9 bigint NOT NULL DEFAULT 0,
+    rt10 bigint NOT NULL DEFAULT 0,
+    rt11 bigint NOT NULL DEFAULT 0,
+    rt12 bigint NOT NULL DEFAULT 0,
+    qw0 bigint NOT NULL DEFAULT 0,
+    qw1 bigint NOT NULL DEFAULT 0,
+    qw2 bigint NOT NULL DEFAULT 0,
+    qw3 bigint NOT NULL DEFAULT 0,
+    qw4 bigint NOT NULL DEFAULT 0,
+    qw5 bigint NOT NULL DEFAULT 0,
+    qw6 bigint NOT NULL DEFAULT 0,
+    qw7 bigint NOT NULL DEFAULT 0,
+    qw8 bigint NOT NULL DEFAULT 0,
+    qw9 bigint NOT NULL DEFAULT 0,
+    qw10 bigint NOT NULL DEFAULT 0,
+    qw11 bigint NOT NULL DEFAULT 0,
+    qw12 bigint NOT NULL DEFAULT 0,
+    PRIMARY KEY (pk, sk)
+);
+CREATE INDEX rollups_expiry_idx ON webhook.rollups (expires_at);
+
+-- Event archive: one gzip JSON Lines batch per row (the replay source and home of large payloads).
+CREATE TABLE webhook.archive_batches (
+    key         text        PRIMARY KEY,
+    body        bytea       NOT NULL,
+    event_count integer     NOT NULL,
+    created_at  timestamptz NOT NULL DEFAULT now(),
+    expires_at  timestamptz NOT NULL
+);
+CREATE INDEX archive_batches_expiry_idx ON webhook.archive_batches (expires_at);
+
+CREATE TABLE webhook.idempotency_keys (
+    tenant_id   text        NOT NULL,
+    key         text        NOT NULL,
+    event_id    text        NOT NULL,
+    expires_at  timestamptz NOT NULL,
+    PRIMARY KEY (tenant_id, key)
+);
+CREATE INDEX idempotency_keys_expiry_idx ON webhook.idempotency_keys (expires_at);
+
+CREATE TABLE webhook.usage_counters (
+    tenant_id  text   NOT NULL,
+    month      text   NOT NULL,
+    accepted   bigint NOT NULL DEFAULT 0,
+    PRIMARY KEY (tenant_id, month)
+);

docs/decisions.md

@@ -55,3 +55,4 @@ Format: ID, date, decision, why. Add new entries at the bottom; do not rewrite h
 - **D51 (2026-10-21) Validate demand before production spend; the product category is real but the independent market is small, so we need a wedge.** Owner asked whether this can be a business (web research 2026-10-21). Findings: the category exists with paying customers (Svix lists Brex, Lob, Lightspeed and Clerk; Hookdeck has a public pricing ladder from a free 10,000 events to $39 and $499 plans), but the independent vendors are small: Svix has raised roughly $2.6M to $13M depending on the source with an estimated 5 to 11 staff and revenue estimates around $1M, Hookdeck about $2.7M raised and 1 to 10 staff (all from low-quality directories, no company-reported revenue). No credible independent market size exists (one report-selling site claims $1.8B growing to $10.2B, not corroborated). The space has free tiers (Hookdeck), open-source and self-hosted options (Svix, Hookdeck Outpost, Hook0), and many companies simply build it themselves or use AWS services. Conclusion: viable as a small, profitable niche business if cost and focus are right, unlikely to be a large market, and not proven for us until a real customer pays. Decisions: (1) Phase L gets demand-validation items L-L12 and L-L13; (2) keep spending near zero (local stack and a staging environment only) until at least one design partner commits, because the production floor is about $200-250 per month (cost-model.md, unverified); (3) pick a wedge from customer conversations; our structural advantage is cost per event, not features. Sources are in the PR description.
 - **D52 (2026-10-21) The project becomes open source (Apache-2.0) and self-hostable with Docker Compose; the hosted paid service is postponed.** Owner changed course after the market review (D51): publish the repo so anyone can use it, goal adoption and reputation, public after a clean-up and a working quick start. Why Apache-2.0: it is the license companies are most comfortable adopting (permissive, patent grant), which matters more for adoption than protecting against cloud vendors; the trade-off is that someone else may sell it as a service, which the owner accepted by choosing adoption over a protected hosted business (AGPL was the alternative). Consequences: Phase O is now the main track; Phase L (company registration, AWS account, payments, legal pages for a paid service) and S-9/S-10 (checkout, self-service signup) are postponed; the developer docs and sample code (S-1..S-5, S-7) move into the public launch; costs drop to zero for the owner. Honest note recorded: open source lowers the barrier to try the product but does not remove the need for a reason to choose it, so the cost per event and one-command simplicity are the wedge (D51).
 - **D53 (2026-10-21) Self-host mode uses PostgreSQL for queues, delivery state, rollups and the event archive; Valkey is optional; AWS adapters stay as an alternative backend.** Why: a self-hoster wants one `docker compose up` with as few moving parts as possible. Today's local stack uses Moto, an AWS test emulator that is not durable or meant for traffic, so it is demo-only. Postgres is already required for configuration (D21), supports a durable queue with `FOR UPDATE SKIP LOCKED` at a few thousand messages per second on one instance, and gives backup/restore for everything in one place. Valkey stays optional for shared rate limits (without it Ingest uses in-process limits, which is exact for one Ingest instance). The event archive defaults to Postgres (compressed batches with a retention job) and can use any S3-compatible store; MinIO is not bundled because of its license and maintenance status. KMS is replaced by a master key from an environment variable or file (envelope encryption, rotation documented); KMS stays available. Trade-offs accepted: a Postgres queue will not reach the 30,000 events/s Stage 2 design (that remains the AWS backend's job, `architecture.md` section 10), and the self-host guide states the supported throughput honestly. Port interfaces in `src/Abstractions` (D37) make this an additional adapter project (`src/Postgres` grows the queue/state/archive adapters), not a rewrite.
+- **D54 (2026-10-21) Self-host backend implemented: `Backend:Mode=Postgres` (migration `0003_selfhost_runtime.sql`, `src/Postgres/Backend`).** Queue: one table `queue_messages` with SQS-like behavior (delay, visibility timeout, five receives then `<queue>-dlq`), receiving with `UPDATE ... WHERE id IN (SELECT ... FOR UPDATE SKIP LOCKED)` so workers never block or double-receive; waiting is a 200 ms poll up to the long-poll time (simple and reliable; `LISTEN/NOTIFY` is a later optimization if latency matters). Why a lease with a receipt UUID: a worker that finished late cannot delete a message that already reappeared and was leased by someone else. State, rollups and archive mirror the DynamoDB/S3 layouts (same column names, same gzip JSONL format, pointer `pg://archive/<key>#<line>`) so documentation and the Athena-style tooling concepts carry over. Idempotency keys and the monthly count live in PostgreSQL by default in this mode (durable across restarts and exact with several Ingest instances) instead of memory; Valkey is therefore optional and only speeds up the rate limit, which stays per-process without it (exact for a single Ingest instance). A maintenance job deletes expired rows hourly (state, archive, rollup minutes, idempotency) and dead-letter messages after 14 days. Retention values are options (`Backend:Postgres:*`). Known limits, stated honestly in the self-host docs: one PostgreSQL instance bounds throughput (target a few thousand events per second; the AWS backend is the path to tens of thousands), the archive grows the database (30-day default retention), and queue polling adds up to ~200 ms latency when idle.

docs/flow-walkthrough.md

@@ -4,6 +4,10 @@ Status: **living document**, kept in step with the code (rule in `CLAUDE.md`, it
 
 Last verified against the code: **2026-10-21** (after the ControlPlane API). The whole path is proven by one test, `tests/IntegrationTests/Aws/PipelineEndToEndTests.cs`: tenant, key and endpoint in PostgreSQL, event posted to Ingest, SQS, Fan-out with S3 archive and routing from the config database, delivery queue, Dispatcher loading the endpoint and its decrypted secret. It uses the local stack (`./scripts/dev-up.sh`); the final HTTP call is a recording sender because loopback targets are blocked by design.
 
+## Backends
+
+The same services run on three backends, selected by `Backend:Mode` (`docs/configuration.md`): `InMemory` (tests, nothing durable), **`Postgres`** (self-host: queues, delivery state, rollups, archive, idempotency and the monthly count all in PostgreSQL, D53/D54; no AWS) and `Aws` (SQS, DynamoDB, S3, KMS; the scale path). The step table below names the AWS adapter files; the PostgreSQL equivalents are in `src/Postgres/Backend`.
+
 ## The path of one event
 
 ```

docs/roadmap.md

@@ -62,9 +62,9 @@ Owner decision: make the repo public (Apache-2.0), goal adoption and reputation,
 | ID | Work item | Size | Notes | Depends |
 |---|---|---|---|---|
 | O-1 | **DONE 2026-10-21** (history scanned: no secrets; remaining owner choices: public contact email, GitHub private vulnerability reporting on) **Public-repo hygiene:** `LICENSE` (Apache-2.0), `NOTICE`, scan the full git history for secrets/personal data, check dependency licenses, `README.md` (what it is, features, architecture picture, quick start, status), `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `SECURITY.md` (how to report), issue and PR templates, repo description and topics | M | do first; blocks going public | |
-| O-2 | **PostgreSQL queue adapters** (`Backend:Mode=Postgres`): durable queue table with visibility timeout, delay, receive count and dead-letter, `SELECT ... FOR UPDATE SKIP LOCKED`; implements the event and delivery publishers/sources and the retry publisher | L | replaces SQS for self-host; target a few thousand messages per second on one Postgres | |
-| O-3 | **PostgreSQL state adapters:** delivery state + due index, rollups (upsert add), idempotency/quota when Valkey is absent | M | replaces DynamoDB | O-2 |
-| O-4 | **Archive adapter without S3:** Postgres (compressed batches) by default with a retention job, S3-compatible storage (any endpoint) optional | M | replaces S3 for self-host; the existing S3 adapter works with any S3-compatible service | |
+| O-2 | **DONE 2026-10-21 (D54)** **PostgreSQL queue adapters** (`Backend:Mode=Postgres`): durable queue table with visibility timeout, delay, receive count and dead-letter, `SELECT ... FOR UPDATE SKIP LOCKED`; implements the event and delivery publishers/sources and the retry publisher | L | replaces SQS for self-host; target a few thousand messages per second on one Postgres | |
+| O-3 | **DONE 2026-10-21 (D54)** **PostgreSQL state adapters:** delivery state + due index, rollups (upsert add), idempotency/quota when Valkey is absent | M | replaces DynamoDB | O-2 |
+| O-4 | **DONE for Postgres archive 2026-10-21 (D54); S3-compatible option exists via the S3 adapter** **Archive adapter without S3:** Postgres (compressed batches) by default with a retention job, S3-compatible storage (any endpoint) optional | M | replaces S3 for self-host; the existing S3 adapter works with any S3-compatible service | |
 | O-5 | **Secrets without KMS:** a production-grade key provider from an environment variable or mounted file (envelope encryption with a master key), documented key rotation; KMS stays optional | S | D45 keeps KMS for AWS | |
 | O-6 | **One-command stack:** `docker compose up` brings up migrator, Ingest, Fan-out, Dispatcher, ControlPlane, PostgreSQL (+ optional Valkey) with generated secrets and healthchecks; prebuilt images published to GitHub Container Registry from CI | M | | O-2..O-5 |
 | O-7 | **Quick start that works in 5 minutes:** create a tenant and key, register an endpoint, send an event, see it arrive (a bundled demo receiver container that verifies the signature and prints events), plus a smoke test script run in CI | M | | O-6 |

docs/worklog.md

@@ -36,6 +36,11 @@ Resume point for agents after a sudden stop, crash, or new session. **Read this
 - **Outcome:** `architecture.md` v4 approved; decisions D20-D28 logged; `contracts.md` v2; `database.md` rewritten config-only (still **in review**); `docs/roadmap.md` lists the work items; plan numbers still provisional.
 - **Resume from:** see the next entry (roadmap execution).
 
+### 2026-10-21 Postgres self-host backend (O-2..O-4)
+- **Status:** PR open
+- **Outcome:** `Backend:Mode=Postgres` with queue, delivery state + due index, rollups, archive, idempotency, quota, maintenance job; hosts wired; tests on real PostgreSQL including the full pipeline without AWS. D54.
+- **Next:** O-5 master key from env/file, network option for private-address delivery (self-hosters call internal services), compose stack + demo receiver + quickstart script (O-6, O-7), README full guide + sample code (O-8, O-11).
+
 ### 2026-10-21 Open source and self-host (Phase O) - current main track
 - **Status:** in progress
 - **Why:** owner decided to publish the repo (Apache-2.0) for adoption and reputation, running on Docker Compose; hosted business postponed (D52, D53). Owner answers: goal adoption, Apache-2.0, real self-host backends, public after clean-up + quick start.

src/Dispatcher/Program.cs

@@ -8,6 +8,7 @@
 using WebhookGateway.Dispatcher.Source;
 using WebhookGateway.Hosting;
 using WebhookGateway.Postgres;
+using WebhookGateway.Postgres.Backend;
 
 var builder = Host.CreateApplicationBuilder(args);
 builder.Logging.AddStructuredJsonConsole("Dispatcher");
@@ -36,12 +37,20 @@
 
 // The consumer needs the queues, the archive and delivery_state, so it only runs with Backend:Mode=Aws
 // (Moto locally, see dev/README.md). The retry scheduler and the rollup flusher run next to it.
-if (builder.Configuration.UseAws())
+if (builder.Configuration.UseAws() || builder.Configuration.UsePostgresBackend())
 {
     builder.Services.Configure<DeliveryOptions>(builder.Configuration.GetSection(DeliveryOptions.SectionName));
     builder.Services.Configure<DeliveryConsumerOptions>(builder.Configuration.GetSection(DeliveryConsumerOptions.SectionName));
-    builder.Services.AddAwsDispatcher(builder.Configuration);
-    builder.Services.AddAwsRollups(builder.Configuration);
+    if (builder.Configuration.UseAws())
+    {
+        builder.Services.AddAwsDispatcher(builder.Configuration);
+        builder.Services.AddAwsRollups(builder.Configuration);
+    }
+    else
+    {
+        builder.Services.AddPostgresDispatcher(builder.Configuration);
+    }
+
     builder.Services.Configure<RollupFlusherOptions>(builder.Configuration.GetSection(RollupFlusherOptions.SectionName));
     builder.Services.AddHostedService<RollupFlusher>();
     builder.Services.AddSingleton<DeliveryProcessor>();
@@ -53,5 +62,5 @@
 
 var host = builder.Build();
 host.Services.GetRequiredService<ILoggerFactory>().CreateLogger("Startup")
-    .LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : "InMemory (no queue consumer)");
+    .LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : builder.Configuration.UsePostgresBackend() ? "Postgres" : "InMemory (no queue consumer)");
 host.Run();

src/FanOut/Program.cs

@@ -8,6 +8,7 @@
 using WebhookGateway.FanOut.Source;
 using WebhookGateway.Hosting;
 using WebhookGateway.Postgres;
+using WebhookGateway.Postgres.Backend;
 
 var builder = Host.CreateApplicationBuilder(args);
 builder.Logging.AddStructuredJsonConsole("FanOut");
@@ -37,7 +38,14 @@
     builder.Services.AddHostedService<RollupFlusher>();
 }
 
+if (builder.Configuration.UsePostgresBackend())
+{
+    builder.Services.AddPostgresFanOut(builder.Configuration);
+    builder.Services.Configure<RollupFlusherOptions>(builder.Configuration.GetSection(RollupFlusherOptions.SectionName));
+    builder.Services.AddHostedService<RollupFlusher>();
+}
+
 var host = builder.Build();
 host.Services.GetRequiredService<ILoggerFactory>().CreateLogger("Startup")
-    .LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : "InMemory");
+    .LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : builder.Configuration.UsePostgresBackend() ? "Postgres" : "InMemory");
 host.Run();

src/Ingest/Program.cs

@@ -5,6 +5,7 @@
 using WebhookGateway.Ingest.Publishing;
 using WebhookGateway.Ingest.Tenancy;
 using WebhookGateway.Postgres;
+using WebhookGateway.Postgres.Backend;
 using WebhookGateway.Valkey;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -33,6 +34,12 @@
     builder.Services.AddSingleton<IApiKeyStore, PostgresApiKeyStore>();
 }
 
+// Backend:Mode=Postgres: events go to a PostgreSQL queue, idempotency keys and the monthly count live in PostgreSQL (D53).
+if (builder.Configuration.UsePostgresBackend())
+{
+    builder.Services.AddPostgresIngest(builder.Configuration);
+}
+
 // With Valkey:ConnectionString set, limits are shared across Ingest tasks; each falls back to local behavior if Valkey is down.
 if (builder.Configuration.UseValkey())
 {
@@ -54,5 +61,5 @@
 app.MapGet("/healthz", () => Results.Ok(new { status = "ok", service = "Ingest" }));
 EventsEndpoint.Map(app);
 
-app.Logger.LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : "InMemory");
+app.Logger.LogServiceStarted(builder.Configuration.UseAws() ? "Aws" : builder.Configuration.UsePostgresBackend() ? "Postgres" : "InMemory");
 app.Run();