Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions internal/app/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"sort"
"strings"

"github.com/compose-spec/compose-go/v2/types"

"github.com/labstack/onebox/internal/shellquote"
)

Expand Down Expand Up @@ -466,14 +468,19 @@ func (p *Spec) renderService(n Names, name string, s Service, selectedImage stri
"labels": map[string]any{"onebox.app": p.Name, "onebox.service": name},
}
}
memory := ""
if s.Resources != nil {
if s.Resources.Memory != "" {
svc["mem_limit"] = s.Resources.Memory
memory = s.Resources.Memory
if memory != "" {
svc["mem_limit"] = memory
}
if s.Resources.CPUs != "" {
svc["cpus"] = s.Resources.CPUs
}
}
if key == "postgres" {
svc["shm_size"] = postgresShmSize(memory)
}

// The whole document is generated, and the shell expansions in it are meant
// for the container, not for Compose on the host.
Expand All @@ -495,6 +502,32 @@ func (p *Spec) renderService(n Names, name string, s Service, selectedImage stri
return append([]byte(header), body...), nil
}

// postgresShmSize sets the POSIX shared-memory ceiling in /dev/shm for parallel
// queries. Docker's default 64 MiB is easily exhausted. This ceiling allocates
// nothing up front, but used pages count against mem_limit, so even the floor
// stays within half.
func postgresShmSize(memory string) int64 {
if memory == "" {
return 1 << 30
}
var limit types.UnitBytes
// Use Compose's memory parser. Zero means unlimited; negative means overflow.
// Leave mem_limit untouched for existing validation.
if err := limit.DecodeMapstructure(memory); err != nil || limit <= 0 {
return 1 << 30
}
shmSize := int64(limit) / 4
if shmSize < 256<<20 {
shmSize = 256 << 20
} else if shmSize > 1<<30 {
shmSize = 1 << 30
}
if shmSize > int64(limit)/2 {
shmSize = int64(limit) / 2
}
return shmSize
}

// identityEnv is the user and database the service is created with, under the
// variable names each driver expects. The user is the driver's fixed role and
// the database is named after the application.
Expand Down
63 changes: 63 additions & 0 deletions internal/app/services_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"path/filepath"
"strings"
"testing"

"gopkg.in/yaml.v3"
)

func serviceSpec(t *testing.T, body string) *Spec {
Expand Down Expand Up @@ -65,6 +67,67 @@ func TestServiceDocumentCarriesNoCredential(t *testing.T) {
}
}

func TestPostgresSharedMemory(t *testing.T) {
for _, tt := range []struct {
name, version, resources, wantMemLimit string
want int64
}{
{"default", "18", "", "", 1 << 30},
{"official image", "17", "", "", 1 << 30},
{"empty resources", "18", ", resources: {}", "", 1 << 30},
{"CPU limit only", "18", ", resources: {cpus: '0.5'}", "", 1 << 30},
{"byte limit", "18", ", resources: {memory: 1B}", "1B", 0},
{"128 MiB limit", "18", ", resources: {memory: 128MB}", "128MB", 64 << 20},
{"256 MiB limit", "18", ", resources: {memory: 256MB}", "256MB", 128 << 20},
{"small limit", "18", ", resources: {memory: 512MB}", "512MB", 256 << 20},
{"zero limit", "18", ", resources: {memory: 0B}", "0B", 1 << 30},
{"floor boundary", "18", ", resources: {memory: 1GB}", "1GB", 256 << 20},
{"mid limit", "18", ", resources: {memory: 2GB}", "2GB", 512 << 20},
{"decimal limit", "18", ", resources: {memory: 1.5GB}", "1.5GB", 384 << 20},
{"cap boundary", "18", ", resources: {memory: 4GB}", "4GB", 1 << 30},
{"large limit", "18", ", resources: {memory: 8GB}", "8GB", 1 << 30},
// The size grammar accepts this, but it overflows Compose's integer limit.
// Rendering must preserve mem_limit for the existing validation path.
{"overflowing limit", "18", ", resources: {memory: 999999999999999999999999TB}", "999999999999999999999999TB", 1 << 30},
} {
t.Run(tt.name, func(t *testing.T) {
doc := renderStore(t, "services: {store: {driver: postgres, version: "+tt.version+tt.resources+"}}\n")
var runtime struct {
Services map[string]struct {
ShmSize int64 `yaml:"shm_size"`
MemLimit string `yaml:"mem_limit"`
} `yaml:"services"`
}
if err := yaml.Unmarshal([]byte(doc), &runtime); err != nil {
t.Fatal(err)
}
service := runtime.Services["store"]
if service.ShmSize != tt.want {
t.Fatalf("shm_size = %d, want %d:\n%s", service.ShmSize, tt.want, doc)
}
if service.MemLimit != tt.wantMemLimit {
t.Fatalf("mem_limit = %q, want %q", service.MemLimit, tt.wantMemLimit)
}
})
}
}

func TestOtherServicesHaveNoSharedMemoryOverride(t *testing.T) {
for _, driver := range DriverNames() {
if driver == "postgres" {
continue
}
t.Run(driver, func(t *testing.T) {
for _, resources := range []string{"", ", resources: {memory: 2GB}"} {
doc := renderStore(t, "services: {store: {driver: "+driver+", version: '1'"+resources+"}}\n")
if strings.Contains(doc, "shm_size") {
t.Fatalf("%s has a shared-memory override:\n%s", driver, doc)
}
}
})
}
}

// A workload that needs a service must be able to reach it and to know how.
func TestNeedingAServiceJoinsItAndReadsItsURL(t *testing.T) {
r, err := serviceSpec(t, "services: {store: {driver: postgres, version: 17}}\n").Resolve("production")
Expand Down
18 changes: 9 additions & 9 deletions internal/app/testdata/contract-verdicts.json
Original file line number Diff line number Diff line change
Expand Up @@ -392,12 +392,12 @@
{
"case": "conformance/service backup policy",
"loads": true,
"digest": "73135190e9f9fae42347e107f496211e4d2f75bfe5ff934ead0096f4f3571a19 postgres=ee9972ab51c56731"
"digest": "73135190e9f9fae42347e107f496211e4d2f75bfe5ff934ead0096f4f3571a19 postgres=ee21ed4fc49969fb"
},
{
"case": "conformance/service scalar",
"loads": true,
"digest": "f6ddf429bf783de806d4b1475df352d4b3ff85210dfe379804248dd3fdd0b80a postgres=f566c9221cb196ab"
"digest": "f6ddf429bf783de806d4b1475df352d4b3ff85210dfe379804248dd3fdd0b80a postgres=ff6eeff3e4bbd9bd"
},
{
"case": "conformance/settings key that is a real driver flag",
Expand Down Expand Up @@ -532,12 +532,12 @@
{
"case": "corpus/authentik.yml",
"loads": true,
"digest": "e82f2e9eac87e3494cf0aa86b80f91a60f363f0af7aac28bb45e1ec6ed2e40fb postgres=d88fd3d620b30ea1 redis=53bad39e8f359a02"
"digest": "e82f2e9eac87e3494cf0aa86b80f91a60f363f0af7aac28bb45e1ec6ed2e40fb postgres=4d380770b261847a redis=53bad39e8f359a02"
},
{
"case": "corpus/ext-authentik-managed.yml",
"loads": true,
"digest": "10748708444a383dc2e587286c13f06a4f3ea218835e2eeab1af83ed312ec741 postgres=d88fd3d620b30ea1 redis=53bad39e8f359a02"
"digest": "10748708444a383dc2e587286c13f06a4f3ea218835e2eeab1af83ed312ec741 postgres=4d380770b261847a redis=53bad39e8f359a02"
},
{
"case": "corpus/ext-authentik.yml",
Expand All @@ -552,7 +552,7 @@
{
"case": "corpus/ext-gitea.yml",
"loads": true,
"digest": "f6028df90e48b43f68c3375115ef402255ebd0aadf0ddde9eb9e2d538598cb65 postgres=446e25618a3d2e4e"
"digest": "f6028df90e48b43f68c3375115ef402255ebd0aadf0ddde9eb9e2d538598cb65 postgres=105be1b298f59e44"
},
{
"case": "corpus/ext-immich-sourced.yml",
Expand All @@ -567,7 +567,7 @@
{
"case": "corpus/ext-n8n.yml",
"loads": true,
"digest": "3831bd9a52d01deada4936e218fc8ba33c96a0b1d939e6bf503f3fbda8ba0895 postgres=c9e06c2c21513656 redis=83f7ec43501bb1f4"
"digest": "3831bd9a52d01deada4936e218fc8ba33c96a0b1d939e6bf503f3fbda8ba0895 postgres=12dc8bcbdff40808 redis=83f7ec43501bb1f4"
},
{
"case": "corpus/ext-paperless.yml",
Expand All @@ -577,12 +577,12 @@
{
"case": "corpus/ext-plausible.yml",
"loads": true,
"digest": "4574810017062b8d857ad2c43597b484fe88e6b4198f75ec74c2d705cbcf17d1 events=79a3b4327069a1dc postgres=128c7ea3ed2b4f69"
"digest": "4574810017062b8d857ad2c43597b484fe88e6b4198f75ec74c2d705cbcf17d1 events=79a3b4327069a1dc postgres=1aea911e88a3631c"
},
{
"case": "corpus/ext-umami.yml",
"loads": true,
"digest": "fd193a8c2f51cbe206e2e038ccd277c3e6785c84912b5967a2afd58c1320ac13 postgres=a0f143f02b9f5845"
"digest": "fd193a8c2f51cbe206e2e038ccd277c3e6785c84912b5967a2afd58c1320ac13 postgres=47bbd25b517473d9"
},
{
"case": "corpus/ghost.yml",
Expand Down Expand Up @@ -622,7 +622,7 @@
{
"case": "corpus/penpot.yml",
"loads": true,
"digest": "1cd500aae4deac5cf744dec3f1222b13aec493457de4b6069a3449126ad70210 postgres=fce3c20f4ad2e0e4 redis=e6f4fb5264fe75ae"
"digest": "1cd500aae4deac5cf744dec3f1222b13aec493457de4b6069a3449126ad70210 postgres=8a2a89524e0749e0 redis=e6f4fb5264fe75ae"
},
{
"case": "corpus/pursue.yml",
Expand Down
21 changes: 16 additions & 5 deletions site/src/content/docs/guides/add-a-database.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,24 @@ That is sufficient as a declaration. Onebox supplies the image, a durable volume
a health check, a credential generated on the server, and the connection details
your application reads. You supply no password, no port, and no data path.

PostgreSQL gets a 1 GiB `/dev/shm` ceiling for parallel queries; with a
positive `resources.memory` limit, it gets one quarter of that limit, bounded
between 256 MiB and 1 GiB, but never more than half the limit. A zero limit means
unlimited. This tmpfs uses memory only as PostgreSQL writes to it, and those
pages count against the container's memory limit.

Declaring it does not start it. Services live outside the release lifecycle, so
`ob deploy` wires up connections but never creates or replaces a service
container — that would make every deploy a potential database restart. Apply the
service explicitly:
`ob deploy` wires up connections. For services with declared extensions, deploy
also converges their configuration and recreates containers when it changes.
Apply a service explicitly with:

```sh
ob service apply --output ndjson
```

A running PostgreSQL service keeps its current `/dev/shm` ceiling until
`ob service apply` recreates it, or deploy does for services with declared extensions.

On a host you have not bootstrapped yet, `ob bootstrap` does this for you. If you
skip it on a live application, the next deploy stops at preflight with
`service "postgres" is not running — start it with ob bootstrap or ob service apply`.
Expand Down Expand Up @@ -165,8 +174,10 @@ is omitted rather than written empty.

## Three properties that always hold

- **A service outlives every release.** It runs in its own Compose project, so no
deploy and no rollback can stop it or remove its volume.
- **A service outlives every release.** It runs in its own Compose project.
Deploy can recreate PostgreSQL services with declared extensions when their
configuration changes; otherwise, deploy and rollback leave services running.
Neither removes their volumes.
- **Its credential is generated on the server, once.** Not in your project, not
in the generated runtime, not in the digest. Never rotated by a re-apply.
- **Its version binds into the release digest**, so a database upgrade under an
Expand Down
Loading