Set the named Playwright executor limit from VM memory

server/cmd/api/api/api.go

@@ -242,7 +242,14 @@ func New(
 		lifecycleCtx:      ctx,
 		lifecycleCancel:   cancel,
 	}
-	s.playwrightExecutors = newPlaywrightExecutorManager(cdpPlaywrightExecutorTabs{withCDP: s.withCDPClient})
+	executorLimit := maxPlaywrightExecutors
+	if mem, err := vmMemoryBytes("/proc/meminfo", "/sys/fs/cgroup/memory.max"); err != nil {
+		slog.Warn("failed to read VM memory; using the maximum playwright executor limit", "error", err, "limit", executorLimit)
+	} else {
+		executorLimit = playwrightExecutorLimit(mem)
+		slog.Info("set playwright executor limit from VM memory", "memory_bytes", mem, "limit", executorLimit)
+	}
+	s.playwrightExecutors = newPlaywrightExecutorManager(cdpPlaywrightExecutorTabs{withCDP: s.withCDPClient}, executorLimit)
 	if err := s.initializeBrowserLocation(); err != nil {
 		cancel()
 		return nil, err

server/cmd/api/api/playwright_executors.go

@@ -12,6 +12,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"slices"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -24,10 +26,19 @@ import (
 const (
 	// defaultPlaywrightExecutor runs calls that name no executor. It always
 	// exists, binds `page` to the active tab instead of owning one, and does
-	// not count toward maxPlaywrightExecutors.
+	// not count toward the named executor limit.
 	defaultPlaywrightExecutor = "default"
 
-	maxPlaywrightExecutors = 8
+	// The named executor limit is set from the VM's memory: what is left after
+	// playwrightExecutorMemoryReserve, divided by playwrightExecutorMemoryBudget,
+	// clamped to [minPlaywrightExecutors, maxPlaywrightExecutors]. The reserve
+	// covers Chromium, the default executor and the API; the budget covers one
+	// executor's daemon process (90-130 MiB RSS) and its tab. A 1 GiB VM gets
+	// 2 named executors; a VM with 2 GiB or more gets the maximum.
+	playwrightExecutorMemoryReserve = 512 << 20
+	playwrightExecutorMemoryBudget  = 192 << 20
+	minPlaywrightExecutors          = 1
+	maxPlaywrightExecutors          = 8
 
 	// playwrightExecutorResponseGrace is added to the execution timeout when
 	// setting the socket deadline, giving the daemon time to report its own
@@ -50,11 +61,52 @@ var (
 )
 
 type playwrightExecutorLimitError struct {
+	limit     int
 	executors []playwrightExecutorInfo
 }
 
 func (e *playwrightExecutorLimitError) Error() string {
-	return fmt.Sprintf("named executor limit (%d) reached; delete one with DELETE /playwright/executors/{name}", maxPlaywrightExecutors)
+	return fmt.Sprintf("named executor limit (%d) reached; delete one with DELETE /playwright/executors/{name}", e.limit)
+}
+
+// playwrightExecutorLimit returns how many named executors fit in memBytes of
+// memory.
+func playwrightExecutorLimit(memBytes uint64) int {
+	if memBytes <= playwrightExecutorMemoryReserve {
+		return minPlaywrightExecutors
+	}
+	n := (memBytes - playwrightExecutorMemoryReserve) / playwrightExecutorMemoryBudget
+	return int(min(max(n, minPlaywrightExecutors), maxPlaywrightExecutors))
+}
+
+// vmMemoryBytes returns the memory available to this VM or container: MemTotal
+// from /proc/meminfo, lowered to the cgroup v2 memory limit when one is set.
+func vmMemoryBytes(meminfoPath, cgroupMaxPath string) (uint64, error) {
+	data, err := os.ReadFile(meminfoPath)
+	if err != nil {
+		return 0, err
+	}
+	var total uint64
+	for line := range strings.Lines(string(data)) {
+		if kb, ok := strings.CutPrefix(line, "MemTotal:"); ok {
+			v, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimSpace(kb), " kB"), 10, 64)
+			if err != nil {
+				return 0, fmt.Errorf("parse MemTotal: %w", err)
+			}
+			total = v << 10
+			break
+		}
+	}
+	if total == 0 {
+		return 0, errors.New("no MemTotal in " + meminfoPath)
+	}
+	// memory.max is "max" when the cgroup has no limit.
+	if data, err := os.ReadFile(cgroupMaxPath); err == nil {
+		if v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64); err == nil {
+			total = min(total, v)
+		}
+	}
+	return total, nil
 }
 
 // playwrightExecutorTabs opens and closes executor tabs. The manager owns
@@ -169,18 +221,21 @@ type playwrightExecutorManager struct {
 	socketDir     string
 	responseGrace time.Duration
 	tabs          playwrightExecutorTabs
+	// limit is how many named executors can exist at once.
+	limit int
 
 	mu        sync.Mutex
 	executors map[string]*playwrightExecutor
 	closed    bool
 }
 
-func newPlaywrightExecutorManager(tabs playwrightExecutorTabs) *playwrightExecutorManager {
+func newPlaywrightExecutorManager(tabs playwrightExecutorTabs, limit int) *playwrightExecutorManager {
 	m := &playwrightExecutorManager{
 		tabs:          tabs,
 		script:        playwrightDaemonScript,
 		socketDir:     os.TempDir(),
 		responseGrace: playwrightExecutorResponseGrace,
+		limit:         limit,
 		executors:     make(map[string]*playwrightExecutor),
 	}
 	e := newPlaywrightExecutor(defaultPlaywrightExecutor)
@@ -212,8 +267,8 @@ func (m *playwrightExecutorManager) executorFor(name string) (*playwrightExecuto
 	if e, ok := m.executors[name]; ok {
 		return e, nil
 	}
-	if m.namedCountLocked() >= maxPlaywrightExecutors {
-		return nil, &playwrightExecutorLimitError{executors: m.snapshotLocked()}
+	if m.namedCountLocked() >= m.limit {
+		return nil, &playwrightExecutorLimitError{limit: m.limit, executors: m.snapshotLocked()}
 	}
 	e := newPlaywrightExecutor(name)
 	m.executors[name] = e

server/cmd/api/api/playwright_executors_test.go

@@ -109,7 +109,7 @@ func newTestPlaywrightExecutorManager(t *testing.T) (*playwrightExecutorManager,
 	tabs := &fakeExecutorTabs{dir: t.TempDir()}
 	t.Setenv("FAKE_TABS_DIR", tabs.dir)
 
-	m := newPlaywrightExecutorManager(tabs)
+	m := newPlaywrightExecutorManager(tabs, maxPlaywrightExecutors)
 	m.script = script
 	m.socketDir = dir
 	m.responseGrace = 500 * time.Millisecond
@@ -348,6 +348,79 @@ func TestPlaywrightExecutorLimit(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestPlaywrightExecutorLimitBelowMax(t *testing.T) {
+	m, _ := newTestPlaywrightExecutorManager(t)
+	m.limit = 2
+	ctx := context.Background()
+
+	for _, name := range []string{"e0", "e1"} {
+		_, err := m.Execute(ctx, name, "sleep:0", 10*time.Second)
+		require.NoError(t, err)
+	}
+	_, err := m.Execute(ctx, "e2", "sleep:0", 10*time.Second)
+	var limitErr *playwrightExecutorLimitError
+	require.ErrorAs(t, err, &limitErr)
+	assert.Contains(t, limitErr.Error(), "named executor limit (2) reached")
+	assert.Len(t, namedExecutors(limitErr.executors), 2)
+
+	_, err = m.Execute(ctx, defaultPlaywrightExecutor, "sleep:0", 10*time.Second)
+	assert.NoError(t, err, "the default executor works at the named executor limit")
+}
+
+func TestPlaywrightExecutorLimitFromMemory(t *testing.T) {
+	const mib = 1 << 20
+	for _, tc := range []struct {
+		name string
+		mem  uint64
+		want int
+	}{
+		{"no memory", 0, minPlaywrightExecutors},
+		{"only the reserve", 512 * mib, minPlaywrightExecutors},
+		{"just over the reserve", 512*mib + 1, minPlaywrightExecutors},
+		{"headless VM", 975 * mib, 2},
+		{"1 GiB VM", 1024 * mib, 2},
+		{"1.5 GiB VM", 1536 * mib, 5},
+		{"just under 2 GiB", 2048*mib - 1, 7},
+		{"2 GiB VM", 2048 * mib, maxPlaywrightExecutors},
+		{"headful VM", 8192 * mib, maxPlaywrightExecutors},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, playwrightExecutorLimit(tc.mem))
+		})
+	}
+}
+
+func TestVMMemoryBytes(t *testing.T) {
+	dir := t.TempDir()
+	meminfo := filepath.Join(dir, "meminfo")
+	cgroupMax := filepath.Join(dir, "memory.max")
+	require.NoError(t, os.WriteFile(meminfo, []byte("MemTotal:         998400 kB\nMemFree:          375808 kB\nMemAvailable:     375808 kB\n"), 0o644))
+	const total = 998400 << 10
+
+	mem, err := vmMemoryBytes(meminfo, cgroupMax)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(total), mem, "no cgroup limit file")
+
+	require.NoError(t, os.WriteFile(cgroupMax, []byte("max\n"), 0o644))
+	mem, err = vmMemoryBytes(meminfo, cgroupMax)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(total), mem, "unlimited cgroup")
+
+	require.NoError(t, os.WriteFile(cgroupMax, []byte("536870912\n"), 0o644))
+	mem, err = vmMemoryBytes(meminfo, cgroupMax)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(512<<20), mem, "a cgroup limit below MemTotal applies")
+
+	require.NoError(t, os.WriteFile(cgroupMax, []byte("17179869184\n"), 0o644))
+	mem, err = vmMemoryBytes(meminfo, cgroupMax)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(total), mem, "a cgroup limit above MemTotal does not apply")
+
+	require.NoError(t, os.WriteFile(meminfo, []byte("MemFree:          375808 kB\n"), 0o644))
+	_, err = vmMemoryBytes(meminfo, cgroupMax)
+	assert.Error(t, err)
+}
+
 func TestPlaywrightExecutorDelete(t *testing.T) {
 	m, _ := newTestPlaywrightExecutorManager(t)
 	ctx := context.Background()