diff --git a/internal/app/services.go b/internal/app/services.go index a9743925..a40c76fd 100644 --- a/internal/app/services.go +++ b/internal/app/services.go @@ -5,6 +5,8 @@ import ( "sort" "strings" + "github.com/compose-spec/compose-go/v2/types" + "github.com/labstack/onebox/internal/shellquote" ) @@ -466,14 +468,19 @@ func (p *Spec) renderService(n Names, name string, s Service, selectedImage stri "labels": map[string]any{"onebox.app": p.Name, "onebox.service": name}, } } + memory := "" if s.Resources != nil { - if s.Resources.Memory != "" { - svc["mem_limit"] = s.Resources.Memory + memory = s.Resources.Memory + if memory != "" { + svc["mem_limit"] = memory } if s.Resources.CPUs != "" { svc["cpus"] = s.Resources.CPUs } } + if key == "postgres" { + svc["shm_size"] = postgresShmSize(memory) + } // The whole document is generated, and the shell expansions in it are meant // for the container, not for Compose on the host. @@ -495,6 +502,32 @@ func (p *Spec) renderService(n Names, name string, s Service, selectedImage stri return append([]byte(header), body...), nil } +// postgresShmSize sets the POSIX shared-memory ceiling in /dev/shm for parallel +// queries. Docker's default 64 MiB is easily exhausted. This ceiling allocates +// nothing up front, but used pages count against mem_limit, so even the floor +// stays within half. +func postgresShmSize(memory string) int64 { + if memory == "" { + return 1 << 30 + } + var limit types.UnitBytes + // Use Compose's memory parser. Zero means unlimited; negative means overflow. + // Leave mem_limit untouched for existing validation. + if err := limit.DecodeMapstructure(memory); err != nil || limit <= 0 { + return 1 << 30 + } + shmSize := int64(limit) / 4 + if shmSize < 256<<20 { + shmSize = 256 << 20 + } else if shmSize > 1<<30 { + shmSize = 1 << 30 + } + if shmSize > int64(limit)/2 { + shmSize = int64(limit) / 2 + } + return shmSize +} + // identityEnv is the user and database the service is created with, under the // variable names each driver expects. The user is the driver's fixed role and // the database is named after the application. diff --git a/internal/app/services_test.go b/internal/app/services_test.go index cd71ff25..74dd1af4 100644 --- a/internal/app/services_test.go +++ b/internal/app/services_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "gopkg.in/yaml.v3" ) func serviceSpec(t *testing.T, body string) *Spec { @@ -65,6 +67,67 @@ func TestServiceDocumentCarriesNoCredential(t *testing.T) { } } +func TestPostgresSharedMemory(t *testing.T) { + for _, tt := range []struct { + name, version, resources, wantMemLimit string + want int64 + }{ + {"default", "18", "", "", 1 << 30}, + {"official image", "17", "", "", 1 << 30}, + {"empty resources", "18", ", resources: {}", "", 1 << 30}, + {"CPU limit only", "18", ", resources: {cpus: '0.5'}", "", 1 << 30}, + {"byte limit", "18", ", resources: {memory: 1B}", "1B", 0}, + {"128 MiB limit", "18", ", resources: {memory: 128MB}", "128MB", 64 << 20}, + {"256 MiB limit", "18", ", resources: {memory: 256MB}", "256MB", 128 << 20}, + {"small limit", "18", ", resources: {memory: 512MB}", "512MB", 256 << 20}, + {"zero limit", "18", ", resources: {memory: 0B}", "0B", 1 << 30}, + {"floor boundary", "18", ", resources: {memory: 1GB}", "1GB", 256 << 20}, + {"mid limit", "18", ", resources: {memory: 2GB}", "2GB", 512 << 20}, + {"decimal limit", "18", ", resources: {memory: 1.5GB}", "1.5GB", 384 << 20}, + {"cap boundary", "18", ", resources: {memory: 4GB}", "4GB", 1 << 30}, + {"large limit", "18", ", resources: {memory: 8GB}", "8GB", 1 << 30}, + // The size grammar accepts this, but it overflows Compose's integer limit. + // Rendering must preserve mem_limit for the existing validation path. + {"overflowing limit", "18", ", resources: {memory: 999999999999999999999999TB}", "999999999999999999999999TB", 1 << 30}, + } { + t.Run(tt.name, func(t *testing.T) { + doc := renderStore(t, "services: {store: {driver: postgres, version: "+tt.version+tt.resources+"}}\n") + var runtime struct { + Services map[string]struct { + ShmSize int64 `yaml:"shm_size"` + MemLimit string `yaml:"mem_limit"` + } `yaml:"services"` + } + if err := yaml.Unmarshal([]byte(doc), &runtime); err != nil { + t.Fatal(err) + } + service := runtime.Services["store"] + if service.ShmSize != tt.want { + t.Fatalf("shm_size = %d, want %d:\n%s", service.ShmSize, tt.want, doc) + } + if service.MemLimit != tt.wantMemLimit { + t.Fatalf("mem_limit = %q, want %q", service.MemLimit, tt.wantMemLimit) + } + }) + } +} + +func TestOtherServicesHaveNoSharedMemoryOverride(t *testing.T) { + for _, driver := range DriverNames() { + if driver == "postgres" { + continue + } + t.Run(driver, func(t *testing.T) { + for _, resources := range []string{"", ", resources: {memory: 2GB}"} { + doc := renderStore(t, "services: {store: {driver: "+driver+", version: '1'"+resources+"}}\n") + if strings.Contains(doc, "shm_size") { + t.Fatalf("%s has a shared-memory override:\n%s", driver, doc) + } + } + }) + } +} + // A workload that needs a service must be able to reach it and to know how. func TestNeedingAServiceJoinsItAndReadsItsURL(t *testing.T) { r, err := serviceSpec(t, "services: {store: {driver: postgres, version: 17}}\n").Resolve("production") diff --git a/internal/app/testdata/contract-verdicts.json b/internal/app/testdata/contract-verdicts.json index d6c95be0..8017da16 100644 --- a/internal/app/testdata/contract-verdicts.json +++ b/internal/app/testdata/contract-verdicts.json @@ -392,12 +392,12 @@ { "case": "conformance/service backup policy", "loads": true, - "digest": "73135190e9f9fae42347e107f496211e4d2f75bfe5ff934ead0096f4f3571a19 postgres=ee9972ab51c56731" + "digest": "73135190e9f9fae42347e107f496211e4d2f75bfe5ff934ead0096f4f3571a19 postgres=ee21ed4fc49969fb" }, { "case": "conformance/service scalar", "loads": true, - "digest": "f6ddf429bf783de806d4b1475df352d4b3ff85210dfe379804248dd3fdd0b80a postgres=f566c9221cb196ab" + "digest": "f6ddf429bf783de806d4b1475df352d4b3ff85210dfe379804248dd3fdd0b80a postgres=ff6eeff3e4bbd9bd" }, { "case": "conformance/settings key that is a real driver flag", @@ -532,12 +532,12 @@ { "case": "corpus/authentik.yml", "loads": true, - "digest": "e82f2e9eac87e3494cf0aa86b80f91a60f363f0af7aac28bb45e1ec6ed2e40fb postgres=d88fd3d620b30ea1 redis=53bad39e8f359a02" + "digest": "e82f2e9eac87e3494cf0aa86b80f91a60f363f0af7aac28bb45e1ec6ed2e40fb postgres=4d380770b261847a redis=53bad39e8f359a02" }, { "case": "corpus/ext-authentik-managed.yml", "loads": true, - "digest": "10748708444a383dc2e587286c13f06a4f3ea218835e2eeab1af83ed312ec741 postgres=d88fd3d620b30ea1 redis=53bad39e8f359a02" + "digest": "10748708444a383dc2e587286c13f06a4f3ea218835e2eeab1af83ed312ec741 postgres=4d380770b261847a redis=53bad39e8f359a02" }, { "case": "corpus/ext-authentik.yml", @@ -552,7 +552,7 @@ { "case": "corpus/ext-gitea.yml", "loads": true, - "digest": "f6028df90e48b43f68c3375115ef402255ebd0aadf0ddde9eb9e2d538598cb65 postgres=446e25618a3d2e4e" + "digest": "f6028df90e48b43f68c3375115ef402255ebd0aadf0ddde9eb9e2d538598cb65 postgres=105be1b298f59e44" }, { "case": "corpus/ext-immich-sourced.yml", @@ -567,7 +567,7 @@ { "case": "corpus/ext-n8n.yml", "loads": true, - "digest": "3831bd9a52d01deada4936e218fc8ba33c96a0b1d939e6bf503f3fbda8ba0895 postgres=c9e06c2c21513656 redis=83f7ec43501bb1f4" + "digest": "3831bd9a52d01deada4936e218fc8ba33c96a0b1d939e6bf503f3fbda8ba0895 postgres=12dc8bcbdff40808 redis=83f7ec43501bb1f4" }, { "case": "corpus/ext-paperless.yml", @@ -577,12 +577,12 @@ { "case": "corpus/ext-plausible.yml", "loads": true, - "digest": "4574810017062b8d857ad2c43597b484fe88e6b4198f75ec74c2d705cbcf17d1 events=79a3b4327069a1dc postgres=128c7ea3ed2b4f69" + "digest": "4574810017062b8d857ad2c43597b484fe88e6b4198f75ec74c2d705cbcf17d1 events=79a3b4327069a1dc postgres=1aea911e88a3631c" }, { "case": "corpus/ext-umami.yml", "loads": true, - "digest": "fd193a8c2f51cbe206e2e038ccd277c3e6785c84912b5967a2afd58c1320ac13 postgres=a0f143f02b9f5845" + "digest": "fd193a8c2f51cbe206e2e038ccd277c3e6785c84912b5967a2afd58c1320ac13 postgres=47bbd25b517473d9" }, { "case": "corpus/ghost.yml", @@ -622,7 +622,7 @@ { "case": "corpus/penpot.yml", "loads": true, - "digest": "1cd500aae4deac5cf744dec3f1222b13aec493457de4b6069a3449126ad70210 postgres=fce3c20f4ad2e0e4 redis=e6f4fb5264fe75ae" + "digest": "1cd500aae4deac5cf744dec3f1222b13aec493457de4b6069a3449126ad70210 postgres=8a2a89524e0749e0 redis=e6f4fb5264fe75ae" }, { "case": "corpus/pursue.yml", diff --git a/site/src/content/docs/guides/add-a-database.mdx b/site/src/content/docs/guides/add-a-database.mdx index fc86ff3d..bf91e5fe 100644 --- a/site/src/content/docs/guides/add-a-database.mdx +++ b/site/src/content/docs/guides/add-a-database.mdx @@ -23,15 +23,24 @@ That is sufficient as a declaration. Onebox supplies the image, a durable volume a health check, a credential generated on the server, and the connection details your application reads. You supply no password, no port, and no data path. +PostgreSQL gets a 1 GiB `/dev/shm` ceiling for parallel queries; with a +positive `resources.memory` limit, it gets one quarter of that limit, bounded +between 256 MiB and 1 GiB, but never more than half the limit. A zero limit means +unlimited. This tmpfs uses memory only as PostgreSQL writes to it, and those +pages count against the container's memory limit. + Declaring it does not start it. Services live outside the release lifecycle, so -`ob deploy` wires up connections but never creates or replaces a service -container — that would make every deploy a potential database restart. Apply the -service explicitly: +`ob deploy` wires up connections. For services with declared extensions, deploy +also converges their configuration and recreates containers when it changes. +Apply a service explicitly with: ```sh ob service apply --output ndjson ``` +A running PostgreSQL service keeps its current `/dev/shm` ceiling until +`ob service apply` recreates it, or deploy does for services with declared extensions. + On a host you have not bootstrapped yet, `ob bootstrap` does this for you. If you skip it on a live application, the next deploy stops at preflight with `service "postgres" is not running — start it with ob bootstrap or ob service apply`. @@ -165,8 +174,10 @@ is omitted rather than written empty. ## Three properties that always hold -- **A service outlives every release.** It runs in its own Compose project, so no - deploy and no rollback can stop it or remove its volume. +- **A service outlives every release.** It runs in its own Compose project. + Deploy can recreate PostgreSQL services with declared extensions when their + configuration changes; otherwise, deploy and rollback leave services running. + Neither removes their volumes. - **Its credential is generated on the server, once.** Not in your project, not in the generated runtime, not in the digest. Never rotated by a re-apply. - **Its version binds into the release digest**, so a database upgrade under an