From d2167a1cdc8d248b2621296e5ff9a0ef029c13b4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:55:26 +0000 Subject: [PATCH] macOS guests 5/6: guest port forwarding Forward HTTP and WebSocket requests from /instances/{id}/ports/{port}/* to a TCP port on a macOS guest's loopback interface, through the guest agent on vsock 2223. Hypeman does not start or supervise services in the guest; the image does. --- cmd/api/api/port_forward.go | 101 +++++++++ cmd/api/api/port_forward_test.go | 221 ++++++++++++++++++ cmd/api/main.go | 24 +- docs/macos-guest-ports.md | 49 ++++ lib/guest/guest.pb.go | 11 +- lib/guest/guest.proto | 2 + lib/guest/port_forward.go | 78 +++++++ lib/guest/port_forward_test.go | 67 ++++++ lib/instances/guest_os.go | 21 +- lib/instances/guest_os_test.go | 12 + lib/scopes/routes_test.go | 6 + lib/scopes/scopes.go | 13 ++ lib/system/guest_agent/agent_info.go | 9 +- lib/system/guest_agent/main.go | 3 + lib/system/guest_agent/port_forward.go | 140 ++++++++++++ lib/system/guest_agent/port_forward_test.go | 238 ++++++++++++++++++++ lib/system/guest_agent/service_test.go | 5 + 17 files changed, 978 insertions(+), 22 deletions(-) create mode 100644 cmd/api/api/port_forward.go create mode 100644 cmd/api/api/port_forward_test.go create mode 100644 docs/macos-guest-ports.md create mode 100644 lib/guest/port_forward.go create mode 100644 lib/guest/port_forward_test.go create mode 100644 lib/system/guest_agent/port_forward.go create mode 100644 lib/system/guest_agent/port_forward_test.go diff --git a/cmd/api/api/port_forward.go b/cmd/api/api/port_forward.go new file mode 100644 index 000000000..596f89aea --- /dev/null +++ b/cmd/api/api/port_forward.go @@ -0,0 +1,101 @@ +package api + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/logger" + mw "github.com/kernel/hypeman/lib/middleware" +) + +// PortForwardHandler forwards HTTP and WebSocket requests on +// /instances/{id}/ports/{port}/* to that TCP port on a macOS guest's loopback +// interface. The Authorization header, hypeman's only credential, is not +// forwarded. Responses come back unchanged; rewriting guest URLs is the caller's job. +// There is no response timeout, so long polls work; a client disconnect ends the forward. +func (s *ApiService) PortForwardHandler(w http.ResponseWriter, r *http.Request) { + inst := mw.GetResolvedInstance[instances.Instance](r.Context()) + if inst == nil { + http.Error(w, `{"code":"internal_error","message":"resource not resolved"}`, http.StatusInternalServerError) + return + } + if reason := inst.GuestOperationUnsupportedReason(instances.GuestOpPortForward); reason != "" { + writeGuestUnsupported(w, reason) + return + } + port, err := strconv.ParseUint(chi.URLParam(r, "port"), 10, 16) + if err != nil || port == 0 { + http.Error(w, `{"code":"invalid_port","message":"port must be between 1 and 65535"}`, http.StatusBadRequest) + return + } + if inst.State != instances.StateRunning { + http.Error(w, fmt.Sprintf(`{"code":"invalid_state","message":"instance must be running (current state: %s)"}`, inst.State), http.StatusConflict) + return + } + path, rawPath, err := forwardedPath(r) + if err != nil { + http.Error(w, `{"code":"invalid_path","message":"invalid escaped path"}`, http.StatusBadRequest) + return + } + dialer, err := s.InstanceManager.GetVsockDialer(r.Context(), inst.Id) + if err != nil { + http.Error(w, `{"code":"unavailable","message":"guest agent unavailable"}`, http.StatusServiceUnavailable) + return + } + + target := net.JoinHostPort("127.0.0.1", strconv.FormatUint(port, 10)) + proxy := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.Out.URL.Scheme = "http" + pr.Out.URL.Host = target + pr.Out.URL.Path = path + pr.Out.URL.RawPath = rawPath + // Guest services such as Chrome DevTools only accept a loopback Host. + pr.Out.Host = target + pr.Out.Header.Del("Authorization") + }, + Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return guest.DialPort(ctx, dialer, uint16(port)) + }, + }, + ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + logger.FromContext(r.Context()).WarnContext(r.Context(), "guest port forward failed", "instance_id", inst.Id, "port", port, "error", err) + if errors.Is(err, guest.ErrGuestPortUnavailable) { + http.Error(w, `{"code":"port_unavailable","message":"nothing is listening on the guest port"}`, http.StatusBadGateway) + return + } + if errors.Is(err, guest.ErrPortForwarderUnavailable) { + http.Error(w, `{"code":"unavailable","message":"guest agent unavailable"}`, http.StatusServiceUnavailable) + return + } + http.Error(w, `{"code":"bad_gateway","message":"guest port forward failed"}`, http.StatusBadGateway) + }, + } + proxy.ServeHTTP(w, r) +} + +// forwardedPath returns the guest path for the wildcard after /ports/{port}. chi +// matches on the escaped path when the request has one, so the wildcard keeps its +// escaping (for example %2F) and is unescaped here rather than escaped twice. +func forwardedPath(r *http.Request) (path, rawPath string, err error) { + suffix := chi.URLParam(r, "*") + if r.URL.RawPath == "" { + return "/" + suffix, "", nil + } + unescaped, err := url.PathUnescape(suffix) + if err != nil { + return "", "", err + } + return "/" + unescaped, "/" + suffix, nil +} diff --git a/cmd/api/api/port_forward_test.go b/cmd/api/api/port_forward_test.go new file mode 100644 index 000000000..5910ed5c0 --- /dev/null +++ b/cmd/api/api/port_forward_test.go @@ -0,0 +1,221 @@ +package api + +import ( + "context" + "encoding/binary" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/gorilla/websocket" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/scopes" + "github.com/stretchr/testify/require" +) + +// forwardingDialer stands in for the guest agent: it answers the port forward +// handshake and connects every forward to backend, whatever port is named. +type forwardingDialer struct { + backend string + ports chan uint16 + dialErr error // the forwarder itself is unreachable + unopened bool // nothing listens on the guest port +} + +func (d *forwardingDialer) DialVsock(_ context.Context, port int) (net.Conn, error) { + if port != guest.PortForwardVsockPort { + return nil, io.ErrUnexpectedEOF + } + if d.dialErr != nil { + return nil, d.dialErr + } + host, agent := net.Pipe() + go func() { + defer agent.Close() + var target uint16 + if err := binary.Read(agent, binary.BigEndian, &target); err != nil { + return + } + d.ports <- target + if d.unopened { + _, _ = agent.Write([]byte{guest.PortForwardStatusDialFailed}) + return + } + conn, err := net.Dial("tcp", d.backend) + if err != nil { + _, _ = agent.Write([]byte{guest.PortForwardStatusDialFailed}) + return + } + defer conn.Close() + _, _ = agent.Write([]byte{guest.PortForwardStatusOK}) + go func() { _, _ = io.Copy(conn, agent) }() + _, _ = io.Copy(agent, conn) + }() + return host, nil +} + +func (d *forwardingDialer) Key() string { return "forwarding" } + +type portForwardManager struct { + instances.Manager + dialer hypervisor.VsockDialer + dialerErr error +} + +func (m portForwardManager) GetVsockDialer(context.Context, string) (hypervisor.VsockDialer, error) { + return m.dialer, m.dialerErr +} + +func newPortForwardServer(t *testing.T, inst *instances.Instance, backend http.Handler, configure ...func(*forwardingDialer, *portForwardManager)) (*httptest.Server, chan uint16) { + t.Helper() + upstream := httptest.NewServer(backend) + t.Cleanup(upstream.Close) + dialer := &forwardingDialer{backend: strings.TrimPrefix(upstream.URL, "http://"), ports: make(chan uint16, 8)} + manager := portForwardManager{dialer: dialer} + for _, c := range configure { + c(dialer, &manager) + } + s := &ApiService{InstanceManager: manager} + r := chi.NewRouter() + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(mw.WithResolvedInstance(r.Context(), inst.Id, inst))) + }) + }) + for _, method := range scopes.PortForwardMethods { + r.MethodFunc(method, scopes.PortForwardRoute, s.PortForwardHandler) + } + server := httptest.NewServer(r) + t.Cleanup(server.Close) + return server, dialer.ports +} + +func runningMacOSInstance() *instances.Instance { + return &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "mac", MacOS: &images.MacOSImage{GuestAgent: true}}, + State: instances.StateRunning, + } +} + +func TestPortForwardForwardsHTTP(t *testing.T) { + var gotPath, gotQuery, gotHost, gotAuth string + server, ports := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotQuery, gotHost, gotAuth = r.URL.Path, r.URL.RawQuery, r.Host, r.Header.Get("Authorization") + _, _ = io.WriteString(w, `{"Browser":"Chrome"}`) + })) + + req, err := http.NewRequest(http.MethodGet, server.URL+"/instances/mac/ports/9222/json/version?x=1", nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer hypeman-token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, `{"Browser":"Chrome"}`, string(body)) + require.EqualValues(t, 9222, <-ports) + require.Equal(t, "/json/version", gotPath) + require.Equal(t, "x=1", gotQuery) + require.Equal(t, "127.0.0.1:9222", gotHost, "guest services see a loopback Host") + require.Empty(t, gotAuth, "hypeman credentials must not reach the guest") +} + +func TestPortForwardForwardsWebSocket(t *testing.T) { + upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + server, _ := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + kind, data, err := conn.ReadMessage() + if err != nil { + return + } + _ = conn.WriteMessage(kind, data) + })) + + conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http")+"/instances/mac/ports/9222/devtools/browser/abc", nil) + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"id":1}`))) + _, data, err := conn.ReadMessage() + require.NoError(t, err) + require.Equal(t, `{"id":1}`, string(data)) +} + +func TestPortForwardRejections(t *testing.T) { + noAgent := runningMacOSInstance() + noAgent.MacOS.GuestAgent = false + stopped := runningMacOSInstance() + stopped.State = instances.StateStopped + linux := &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "linux"}, State: instances.StateRunning} + + for _, tc := range []struct { + name string + inst *instances.Instance + port string + code int + }{ + {"linux guest", linux, "9222", http.StatusNotImplemented}, + {"macOS without agent", noAgent, "9222", http.StatusNotImplemented}, + {"port zero", runningMacOSInstance(), "0", http.StatusBadRequest}, + {"port out of range", runningMacOSInstance(), "70000", http.StatusBadRequest}, + {"stopped", stopped, "9222", http.StatusConflict}, + } { + t.Run(tc.name, func(t *testing.T) { + server, _ := newPortForwardServer(t, tc.inst, http.NotFoundHandler()) + resp, err := http.Get(server.URL + "/instances/" + tc.inst.Id + "/ports/" + tc.port + "/json") + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, tc.code, resp.StatusCode) + }) + } +} + +func TestPortForwardKeepsEscapedPath(t *testing.T) { + var gotRawPath string + server, _ := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotRawPath = r.URL.EscapedPath() + })) + resp, err := http.Get(server.URL + "/instances/mac/ports/9222/a%2Fb/c%20d") + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "/a%2Fb/c%20d", gotRawPath, "escapes reach the guest once, not twice") +} + +func TestPortForwardForwardFailures(t *testing.T) { + for _, tc := range []struct { + name string + configure func(*forwardingDialer, *portForwardManager) + code int + body string + }{ + {"nothing listening", func(d *forwardingDialer, _ *portForwardManager) { d.unopened = true }, http.StatusBadGateway, "port_unavailable"}, + {"forwarder unreachable", func(d *forwardingDialer, _ *portForwardManager) { d.dialErr = errors.New("connection refused") }, http.StatusServiceUnavailable, "unavailable"}, + {"agent disabled", func(_ *forwardingDialer, m *portForwardManager) { m.dialerErr = instances.ErrInvalidRequest }, http.StatusServiceUnavailable, "unavailable"}, + } { + t.Run(tc.name, func(t *testing.T) { + server, _ := newPortForwardServer(t, runningMacOSInstance(), http.NotFoundHandler(), tc.configure) + resp, err := http.Get(server.URL + "/instances/mac/ports/9222/json") + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, err) + require.Equal(t, tc.code, resp.StatusCode) + require.Contains(t, string(body), tc.body) + }) + } +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 47bc41a3a..b30a0434f 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -447,9 +447,10 @@ func run() error { // See: https://github.com/oapi-codegen/nethttp-middleware#usage spec.Servers = nil - // Custom exec endpoint (outside OpenAPI spec, uses WebSocket) + // Exec and cp (WebSocket) and port forwarding (HTTP and WebSocket proxy) are + // outside the OpenAPI spec and share one middleware chain. // Note: No otelchi here as WebSocket doesn't work well with tracing middleware - r.With( + guestRoutes := r.With( middleware.RequestID, middleware.RealIP, middleware.Recoverer, @@ -458,19 +459,12 @@ func run() error { mw.JwtAuth(app.Config.JwtSecret), scopes.RequireScope(scopes.InstanceWrite), mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), - ).Get("/instances/{id}/exec", app.ApiService.ExecHandler) - - // Custom cp endpoint (outside OpenAPI spec, uses WebSocket) - r.With( - middleware.RequestID, - middleware.RealIP, - middleware.Recoverer, - mw.InjectLogger(logger), - mw.AccessLogger(accessLogger), - mw.JwtAuth(app.Config.JwtSecret), - scopes.RequireScope(scopes.InstanceWrite), - mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), - ).Get("/instances/{id}/cp", app.ApiService.CpHandler) + ) + guestRoutes.Get("/instances/{id}/exec", app.ApiService.ExecHandler) + guestRoutes.Get("/instances/{id}/cp", app.ApiService.CpHandler) + for _, method := range scopes.PortForwardMethods { + guestRoutes.MethodFunc(method, scopes.PortForwardRoute, app.ApiService.PortForwardHandler) + } // Create builder VM resolver for secure token authentication // This validates that token requests from builder VMs are for their authorized repos only diff --git a/docs/macos-guest-ports.md b/docs/macos-guest-ports.md new file mode 100644 index 000000000..152cbfc05 --- /dev/null +++ b/docs/macos-guest-ports.md @@ -0,0 +1,49 @@ +# macOS guest port forwarding + +`/instances/{id}/ports/{port}/*` forwards HTTP and WebSocket requests through the +guest agent (vsock 2223) to `127.0.0.1:{port}` in a running macOS guest. It +replaces ingress, which macOS guests do not support because their addresses are +vmnet NAT leases. + +## Request handling + +- The route requires the same `instance:write` scope as exec. Exec already runs + commands as root in the guest, so the forwarder does not restrict which loopback + port it reaches. +- The path after `/ports/{port}` and the query string are forwarded unchanged. + The `Host` header is rewritten to `127.0.0.1:{port}`, because services such as + Chrome DevTools reject other hosts. The `Authorization` header is removed. +- Responses come back unchanged. URLs a guest service returns (for example + `webSocketDebuggerUrl` from `/json/version`) still name the guest's loopback + address. Rewriting them for an external client is the caller's job. +- `GET`, `HEAD`, `POST`, `PUT`, `PATCH` and `DELETE` are forwarded. WebSocket + upgrades are forwarded as ordinary `GET` requests. +- There is no response timeout, so long polls and slow responses work. A client + disconnect ends the forward. + +## Errors + +| Status | Cause | +| --- | --- | +| 501 | Linux guest, a macOS image that does not declare the guest agent, or an agent that does not report the port-forward capability | +| 400 | Port outside 1-65535 | +| 409 | Instance is not running (it is `Initializing` until the guest agent is ready) | +| 502 | Nothing is listening on the guest port, or the forward failed | +| 503 | Guest agent unavailable or forwarding limit reached | + +## Guest side + +The guest agent listens on vsock 2223 only on Darwin. Each host connection +starts with the target port as two big-endian bytes. The agent connects to that +port on `127.0.0.1` and answers with one status byte (0 connected, 1 failed) +before copying bytes both ways. Either side closing ends the forward. + +Each guest agent admits at most 64 concurrent forwards, including handshakes. +Excess connections are closed immediately, without dialing the guest service +or creating handler goroutines. Handshakes, including status writes, have a +five-second deadline. Closing a forward releases its slot. + +Hypeman does not start or supervise services inside the guest. A browser, its +flags and its profile belong to the guest image, for example as a LaunchAgent in +the logged-in user's session that starts the browser with +`--remote-debugging-port`. Hypeman only forwards to the port the image chose. diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index a5970eb34..4dba17607 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -77,15 +77,19 @@ type AgentCapability int32 const ( AgentCapability_AGENT_CAPABILITY_UNSPECIFIED AgentCapability = 0 + // HTTP and WebSocket forwarding to guest loopback ports over vsock 2223 + AgentCapability_AGENT_CAPABILITY_PORT_FORWARD AgentCapability = 1 ) // Enum value maps for AgentCapability. var ( AgentCapability_name = map[int32]string{ 0: "AGENT_CAPABILITY_UNSPECIFIED", + 1: "AGENT_CAPABILITY_PORT_FORWARD", } AgentCapability_value = map[string]int32{ - "AGENT_CAPABILITY_UNSPECIFIED": 0, + "AGENT_CAPABILITY_UNSPECIFIED": 0, + "AGENT_CAPABILITY_PORT_FORWARD": 1, } ) @@ -1779,9 +1783,10 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\fGPUInitState\x12\x1a\n" + "\x16GPU_INIT_STATE_UNKNOWN\x10\x00\x12\x15\n" + "\x11GPU_INIT_STATE_OK\x10\x01\x12\x19\n" + - "\x15GPU_INIT_STATE_FAILED\x10\x02*3\n" + + "\x15GPU_INIT_STATE_FAILED\x10\x02*V\n" + "\x0fAgentCapability\x12 \n" + - "\x1cAGENT_CAPABILITY_UNSPECIFIED\x10\x002\xcc\x04\n" + + "\x1cAGENT_CAPABILITY_UNSPECIFIED\x10\x00\x12!\n" + + "\x1dAGENT_CAPABILITY_PORT_FORWARD\x10\x012\xcc\x04\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 9b59e4a4a..4c394860d 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -198,6 +198,8 @@ message GetGPUInitStatusResponse { // such as exec, copy, stat and shutdown, are not listed. enum AgentCapability { AGENT_CAPABILITY_UNSPECIFIED = 0; + // HTTP and WebSocket forwarding to guest loopback ports over vsock 2223 + AGENT_CAPABILITY_PORT_FORWARD = 1; } // GetAgentInfoRequest requests the agent's version information diff --git a/lib/guest/port_forward.go b/lib/guest/port_forward.go new file mode 100644 index 000000000..ddb0477f7 --- /dev/null +++ b/lib/guest/port_forward.go @@ -0,0 +1,78 @@ +package guest + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PortForwardVsockPort is the guest vsock port where the guest agent accepts +// host connections to forward to a TCP port on the guest's loopback interface. +const PortForwardVsockPort = 2223 + +// Port forward handshake: the host sends the target port as two big-endian bytes, +// and the agent answers with one status byte before copying bytes both ways. +const ( + PortForwardStatusOK byte = 0 + PortForwardStatusDialFailed byte = 1 +) + +// ErrGuestPortUnavailable means nothing accepted the connection on the guest port. +var ErrGuestPortUnavailable = errors.New("guest port unavailable") + +// ErrPortForwarderUnavailable means the guest agent's port forwarder did not answer. +var ErrPortForwarderUnavailable = errors.New("guest port forwarder unavailable") + +// portForwardHandshakeTimeout bounds the handshake, including the agent's dial. +const portForwardHandshakeTimeout = 10 * time.Second + +// DialPort opens a connection to TCP port on the guest's loopback interface, +// through the guest agent's port forwarder. ctx bounds only the handshake. +func DialPort(ctx context.Context, dialer hypervisor.VsockDialer, port uint16) (net.Conn, error) { + if port == 0 { + return nil, fmt.Errorf("invalid guest port 0") + } + conn, err := dialer.DialVsock(ctx, PortForwardVsockPort) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrPortForwarderUnavailable, err) + } + // Cancellation unblocks the handshake by closing the connection. + stop := context.AfterFunc(ctx, func() { conn.Close() }) + err = portForwardHandshake(conn, port) + if !stop() { + conn.Close() + return nil, ctx.Err() + } + if err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +func portForwardHandshake(conn net.Conn, port uint16) error { + if err := conn.SetDeadline(time.Now().Add(portForwardHandshakeTimeout)); err != nil { + return fmt.Errorf("%w: set handshake deadline: %v", ErrPortForwarderUnavailable, err) + } + var status [1]byte + err := binary.Write(conn, binary.BigEndian, port) + if err == nil { + _, err = io.ReadFull(conn, status[:]) + } + if err != nil { + return fmt.Errorf("%w: handshake: %v", ErrPortForwarderUnavailable, err) + } + if status[0] != PortForwardStatusOK { + return fmt.Errorf("%w: %d", ErrGuestPortUnavailable, port) + } + if err := conn.SetDeadline(time.Time{}); err != nil { + return fmt.Errorf("%w: clear handshake deadline: %v", ErrPortForwarderUnavailable, err) + } + return nil +} diff --git a/lib/guest/port_forward_test.go b/lib/guest/port_forward_test.go new file mode 100644 index 000000000..a1864f9ed --- /dev/null +++ b/lib/guest/port_forward_test.go @@ -0,0 +1,67 @@ +package guest + +import ( + "context" + "encoding/binary" + "io" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +type pipeDialer struct { + serve func(net.Conn) + port int +} + +func (d *pipeDialer) DialVsock(_ context.Context, port int) (net.Conn, error) { + d.port = port + host, agent := net.Pipe() + go d.serve(agent) + return host, nil +} + +func (d *pipeDialer) Key() string { return "pipe" } + +func answerPortForward(status byte, gotPort *uint16) func(net.Conn) { + return func(agent net.Conn) { + defer agent.Close() + if err := binary.Read(agent, binary.BigEndian, gotPort); err != nil { + return + } + if _, err := agent.Write([]byte{status}); err != nil || status != PortForwardStatusOK { + return + } + _, _ = io.Copy(agent, agent) + } +} + +func TestDialPortHandshake(t *testing.T) { + var gotPort uint16 + dialer := &pipeDialer{serve: answerPortForward(PortForwardStatusOK, &gotPort)} + conn, err := DialPort(context.Background(), dialer, 9222) + require.NoError(t, err) + defer conn.Close() + require.Equal(t, PortForwardVsockPort, dialer.port) + require.EqualValues(t, 9222, gotPort) + + _, err = conn.Write([]byte("ping")) + require.NoError(t, err) + buf := make([]byte, 4) + _, err = io.ReadFull(conn, buf) + require.NoError(t, err) + require.Equal(t, "ping", string(buf)) +} + +func TestDialPortReportsUnavailablePort(t *testing.T) { + var gotPort uint16 + dialer := &pipeDialer{serve: answerPortForward(PortForwardStatusDialFailed, &gotPort)} + _, err := DialPort(context.Background(), dialer, 9222) + require.ErrorIs(t, err, ErrGuestPortUnavailable) +} + +func TestDialPortRejectsPortZero(t *testing.T) { + _, err := DialPort(context.Background(), &pipeDialer{}, 0) + require.Error(t, err) +} diff --git a/lib/instances/guest_os.go b/lib/instances/guest_os.go index 647d4e303..7d5c4c637 100644 --- a/lib/instances/guest_os.go +++ b/lib/instances/guest_os.go @@ -1,6 +1,10 @@ package instances -import "fmt" +import ( + "fmt" + + "github.com/kernel/hypeman/lib/guest" +) // GuestOperation names a host API operation whose availability depends on the // guest OS and on the shared guest agent. @@ -10,6 +14,9 @@ const ( GuestOpExec GuestOperation = "exec" GuestOpFileCopy GuestOperation = "file copy" GuestOpStat GuestOperation = "stat" + // GuestOpPortForward reaches a TCP port on a macOS guest's loopback interface. + // Linux guests expose ports through ingress instead. + GuestOpPortForward GuestOperation = "port forwarding" ) // GuestOperationUnsupportedReason returns an empty string when op is available for @@ -17,8 +24,18 @@ const ( // which operations need the shared system agent. Readiness, probes and shutdown check // GuestAgentEnabled directly, which is the same predicate. func (s *StoredMetadata) GuestOperationUnsupportedReason(op GuestOperation) string { - if s.MacOS != nil && !s.GuestAgentEnabled() { + if s.MacOS == nil { + if op == GuestOpPortForward { + return "port forwarding is only available for macOS guests; use ingress for Linux guests" + } + return "" + } + if !s.GuestAgentEnabled() { return fmt.Sprintf("%s requires the shared guest agent on macOS instances", op) } + // An agent recorded before GetAgentInfo existed is not checked; the next boot records it. + if op == GuestOpPortForward && s.GuestAgent != nil && !s.GuestAgent.HasCapability(guest.AgentCapability_AGENT_CAPABILITY_PORT_FORWARD) { + return fmt.Sprintf("the guest agent in this image (%q, protocol %d) does not support port forwarding; use an image with a newer agent", s.GuestAgent.Version, s.GuestAgent.ProtocolVersion) + } return "" } diff --git a/lib/instances/guest_os_test.go b/lib/instances/guest_os_test.go index 054fa0f69..2ce9f135e 100644 --- a/lib/instances/guest_os_test.go +++ b/lib/instances/guest_os_test.go @@ -3,6 +3,7 @@ package instances import ( "testing" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/images" "github.com/stretchr/testify/require" ) @@ -21,3 +22,14 @@ func TestGuestOperationCapabilityTable(t *testing.T) { require.Empty(t, withAgent.GuestOperationUnsupportedReason(op)) } } + +func TestPortForwardRequiresAgentCapability(t *testing.T) { + agent := func(info *GuestAgentInfo) *StoredMetadata { + return &StoredMetadata{MacOS: &images.MacOSImage{GuestAgent: true}, GuestAgent: info} + } + require.Empty(t, agent(&GuestAgentInfo{ProtocolVersion: 1, Capabilities: []string{guest.AgentCapability_AGENT_CAPABILITY_PORT_FORWARD.String()}}).GuestOperationUnsupportedReason(GuestOpPortForward)) + require.Contains(t, agent(&GuestAgentInfo{}).GuestOperationUnsupportedReason(GuestOpPortForward), "does not support port forwarding", "an agent that predates the capability is refused") + require.Empty(t, agent(&GuestAgentInfo{}).GuestOperationUnsupportedReason(GuestOpExec), "other operations do not need it") + require.Empty(t, agent(nil).GuestOperationUnsupportedReason(GuestOpPortForward), "an agent recorded before GetAgentInfo is not checked") + require.Contains(t, (&StoredMetadata{}).GuestOperationUnsupportedReason(GuestOpPortForward), "use ingress") +} diff --git a/lib/scopes/routes_test.go b/lib/scopes/routes_test.go index f78bf3f89..eb1ba4a62 100644 --- a/lib/scopes/routes_test.go +++ b/lib/scopes/routes_test.go @@ -23,6 +23,9 @@ func TestAllRoutesHaveScopes(t *testing.T) { // WebSocket endpoints registered outside OpenAPI (same as cmd/api/main.go) r.Get("/instances/{id}/exec", noop) r.Get("/instances/{id}/cp", noop) + for _, method := range scopes.PortForwardMethods { + r.MethodFunc(method, scopes.PortForwardRoute, noop) + } // Public/unauthenticated endpoints r.Get("/spec.yaml", noop) @@ -88,6 +91,9 @@ func TestRouteScopesHaveNoStaleEntries(t *testing.T) { // Mirror production routes r.Get("/instances/{id}/exec", noop) r.Get("/instances/{id}/cp", noop) + for _, method := range scopes.PortForwardMethods { + r.MethodFunc(method, scopes.PortForwardRoute, noop) + } r.Get("/spec.yaml", noop) r.Get("/spec.json", noop) r.Get("/swagger", noop) diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index 9d078c1fb..d19a11798 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -198,6 +198,19 @@ var DirectScopeRoutes = map[string]Scope{ "GET /instances/{id}/cp": InstanceWrite, } +// PortForwardRoute forwards HTTP and WebSocket requests to a port on a macOS guest. +// It needs the same scope as exec, for each method in PortForwardMethods. +const PortForwardRoute = "/instances/{id}/ports/{port}/*" + +// PortForwardMethods are the HTTP methods PortForwardRoute accepts. +var PortForwardMethods = []string{http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete} + +func init() { + for _, method := range PortForwardMethods { + DirectScopeRoutes[method+" "+PortForwardRoute] = InstanceWrite + } +} + // RouteScopes maps "METHOD /path-pattern" to the required scope. // Path patterns use chi-style {param} placeholders. var RouteScopes = map[string]Scope{ diff --git a/lib/system/guest_agent/agent_info.go b/lib/system/guest_agent/agent_info.go index 2b87a2c25..f2365c7d0 100644 --- a/lib/system/guest_agent/agent_info.go +++ b/lib/system/guest_agent/agent_info.go @@ -15,12 +15,17 @@ const agentProtocolVersion = 1 var agentVersion = "dev" // agentCapabilities lists optional features this agent supports. -var agentCapabilities []pb.AgentCapability +func agentCapabilities() []pb.AgentCapability { + if portForwardSupported { + return []pb.AgentCapability{pb.AgentCapability_AGENT_CAPABILITY_PORT_FORWARD} + } + return nil +} func (s *guestServer) GetAgentInfo(context.Context, *pb.GetAgentInfoRequest) (*pb.GetAgentInfoResponse, error) { return &pb.GetAgentInfoResponse{ ProtocolVersion: agentProtocolVersion, Version: agentVersion, - Capabilities: agentCapabilities, + Capabilities: agentCapabilities(), }, nil } diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 7d2633538..d3c93cb55 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -50,6 +50,9 @@ func main() { defer l.Close() log.Println("[guest-agent] listening on vsock port 2222") + if portForwardSupported { + startPortForwards() + } log.Printf("[guest-agent] %s ts=%s", readySentinelPrefix, time.Now().UTC().Format(time.RFC3339Nano)) if err := signalReadyFD(); err != nil { log.Printf("[guest-agent] warning: failed to signal readiness fd: %v", err) diff --git a/lib/system/guest_agent/port_forward.go b/lib/system/guest_agent/port_forward.go new file mode 100644 index 000000000..44bb95d01 --- /dev/null +++ b/lib/system/guest_agent/port_forward.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "io" + "log" + "net" + "runtime" + "strconv" + "time" +) + +const maxConcurrentPortForwards = 64 + +// Port forward protocol, matching lib/guest/port_forward.go. The builder image +// compiles this agent with only the generated files from lib/guest. +const ( + portForwardVsockPort = 2223 + portForwardStatusOK = byte(0) + portForwardStatusDialFailed = byte(1) +) + +// portForwardSupported is true where the agent forwards guest ports. Linux guests +// reach their ports through ingress instead. +const portForwardSupported = runtime.GOOS == "darwin" + +// startPortForwards listens on the port forward vsock port and connects each +// host connection to the TCP port it names on the guest's loopback interface. +// The host API requires instance-write access, which also grants exec, so the +// forwarder does not restrict which loopback ports it reaches. +func startPortForwards() { + go func() { + l := listenWithRetry(func() (net.Listener, error) { return listenVsock(portForwardVsockPort) }, time.Second, 30*time.Second) + log.Printf("[guest-agent] port forwarding on vsock port %d", portForwardVsockPort) + servePortForwards(l, dialLoopback) + }() +} + +// listenWithRetry retries listen until it succeeds. The agent reports ready +// without waiting for the forwarder, so giving up would leave forwarding off for +// the rest of the agent's life. +func listenWithRetry(listen func() (net.Listener, error), backoff, maxBackoff time.Duration) net.Listener { + for attempt := 1; ; attempt++ { + l, err := listen() + if err == nil { + return l + } + log.Printf("[guest-agent] port forward listen attempt %d failed: %v", attempt, err) + time.Sleep(backoff) + backoff = min(2*backoff, maxBackoff) + } +} + +// servePortForwards accepts until the listener closes. Other accept errors, such as +// running out of file descriptors, are retried so forwarding recovers. +func servePortForwards(l net.Listener, dial func(context.Context, uint16) (net.Conn, error)) { + defer l.Close() + slots := make(chan struct{}, maxConcurrentPortForwards) + backoff := 5 * time.Millisecond + for { + conn, err := l.Accept() + if errors.Is(err, net.ErrClosed) { + return + } + if err != nil { + log.Printf("[guest-agent] port forward accept: %v", err) + time.Sleep(backoff) + backoff = min(2*backoff, time.Second) + continue + } + backoff = 5 * time.Millisecond + select { + case slots <- struct{}{}: + go func() { + defer func() { <-slots }() + forwardGuestPort(conn, dial) + }() + default: + conn.Close() + } + } +} + +// portForwardStepTimeout bounds each handshake step: reading the port, dialing it, +// and writing the status. The host allows two steps for the whole handshake. Tests +// shorten it. +var portForwardStepTimeout = 5 * time.Second + +func dialLoopback(ctx context.Context, port uint16) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(int(port)))) +} + +// forwardGuestPort runs the handshake for one host connection, then copies bytes +// between it and the loopback port until either side closes. +func forwardGuestPort(host net.Conn, dial func(context.Context, uint16) (net.Conn, error)) { + defer host.Close() + _ = host.SetDeadline(time.Now().Add(portForwardStepTimeout)) + var port uint16 + if err := binary.Read(host, binary.BigEndian, &port); err != nil { + return + } + + if port == 0 { + _, _ = host.Write([]byte{portForwardStatusDialFailed}) + return + } + ctx, cancel := context.WithTimeout(context.Background(), portForwardStepTimeout) + target, err := dial(ctx, port) + cancel() + // A slow dial must not use up the time for reporting its result, or the host + // sees a dropped handshake (503) instead of a failed dial (502). + _ = host.SetWriteDeadline(time.Now().Add(portForwardStepTimeout)) + if err != nil { + _, _ = host.Write([]byte{portForwardStatusDialFailed}) + return + } + defer target.Close() + if _, err := host.Write([]byte{portForwardStatusOK}); err != nil { + return + } + _ = host.SetDeadline(time.Time{}) + + done := make(chan struct{}, 2) + go func() { + _, _ = io.Copy(target, host) + done <- struct{}{} + }() + go func() { + _, _ = io.Copy(host, target) + done <- struct{}{} + }() + // Either side closing ends the forward; closing both unblocks the other copy. + <-done + host.Close() + target.Close() + <-done +} diff --git a/lib/system/guest_agent/port_forward_test.go b/lib/system/guest_agent/port_forward_test.go new file mode 100644 index 000000000..3bcbda55a --- /dev/null +++ b/lib/system/guest_agent/port_forward_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "sync/atomic" + "testing" + "time" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/stretchr/testify/require" +) + +// startLoopbackEcho serves one echo connection on loopback and returns its port. +func startLoopbackEcho(t *testing.T) uint16 { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { l.Close() }) + go func() { + conn, err := l.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = io.Copy(conn, conn) + }() + return uint16(l.Addr().(*net.TCPAddr).Port) +} + +func TestForwardGuestPortCopiesBothWays(t *testing.T) { + port := startLoopbackEcho(t) + + host, agent := net.Pipe() + done := make(chan struct{}) + go func() { + forwardGuestPort(agent, dialLoopback) + close(done) + }() + require.NoError(t, binary.Write(host, binary.BigEndian, port)) + status := make([]byte, 1) + _, err := io.ReadFull(host, status) + require.NoError(t, err) + require.Equal(t, portForwardStatusOK, status[0]) + + _, err = host.Write([]byte("hello")) + require.NoError(t, err) + buf := make([]byte, 5) + _, err = io.ReadFull(host, buf) + require.NoError(t, err) + require.Equal(t, "hello", string(buf)) + + require.NoError(t, host.Close()) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("forward did not end after the host closed") + } +} + +func TestForwardGuestPortReportsDialFailure(t *testing.T) { + for _, port := range []uint16{0, 9222} { + t.Run(fmt.Sprint(port), func(t *testing.T) { + host, agent := net.Pipe() + defer host.Close() + refuse := func(context.Context, uint16) (net.Conn, error) { return nil, fmt.Errorf("connection refused") } + go forwardGuestPort(agent, refuse) + require.NoError(t, binary.Write(host, binary.BigEndian, port)) + status := make([]byte, 1) + _, err := io.ReadFull(host, status) + require.NoError(t, err) + require.Equal(t, portForwardStatusDialFailed, status[0]) + }) + } +} + +func TestServePortForwardsBoundsConcurrencyAndReusesSlots(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { listener.Close() }) + peers := make(chan net.Conn, maxConcurrentPortForwards+1) + var dialed atomic.Int32 + dial := func(context.Context, uint16) (net.Conn, error) { + a, b := net.Pipe() + peers <- b + dialed.Add(1) + return a, nil + } + done := make(chan struct{}) + go func() { servePortForwards(listener, dial); close(done) }() + openForward := func() (net.Conn, error) { + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if err != nil { + return nil, err + } + _ = conn.SetDeadline(time.Now().Add(time.Second)) + var status [1]byte + err = binary.Write(conn, binary.BigEndian, uint16(9222)) + if err == nil { + _, err = io.ReadFull(conn, status[:]) + } + if err != nil || status[0] != portForwardStatusOK { + conn.Close() + return nil, fmt.Errorf("handshake failed: %v", err) + } + return conn, nil + } + var hosts []net.Conn + t.Cleanup(func() { + for _, conn := range hosts { + conn.Close() + } + listener.Close() + <-done + for len(peers) > 0 { + (<-peers).Close() + } + }) + for range maxConcurrentPortForwards { + conn, err := openForward() + require.NoError(t, err) + hosts = append(hosts, conn) + } + _, err = openForward() + require.Error(t, err, "excess forwards must be rejected, not queued") + require.Equal(t, int32(maxConcurrentPortForwards), dialed.Load(), "overload must not dial the target") + + require.NoError(t, hosts[0].Close()) + require.Eventually(t, func() bool { + conn, err := openForward() + if err != nil { + return false + } + hosts = append(hosts, conn) + return true + }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, int32(maxConcurrentPortForwards+1), dialed.Load()) +} + +func TestForwardGuestPortBoundsStatusWrite(t *testing.T) { + host, agent := net.Pipe() + defer host.Close() + done := make(chan struct{}) + go func() { forwardGuestPort(agent, dialLoopback); close(done) }() + require.NoError(t, binary.Write(host, binary.BigEndian, uint16(0))) + select { + case <-done: + case <-time.After(7 * time.Second): + t.Fatal("an unread handshake status must not hold a forwarding slot indefinitely") + } +} + +func TestForwardGuestPortReportsSlowDialFailure(t *testing.T) { + prev := portForwardStepTimeout + portForwardStepTimeout = 200 * time.Millisecond + t.Cleanup(func() { portForwardStepTimeout = prev }) + + host, agent := net.Pipe() + defer host.Close() + // The dial outlasts its timeout, as a slow-to-refuse target can. + slowDial := func(ctx context.Context, _ uint16) (net.Conn, error) { + <-ctx.Done() + time.Sleep(portForwardStepTimeout / 2) + return nil, ctx.Err() + } + go forwardGuestPort(agent, slowDial) + require.NoError(t, binary.Write(host, binary.BigEndian, uint16(8080))) + status := make([]byte, 1) + _, err := io.ReadFull(host, status) + require.NoError(t, err, "a dial that uses its whole timeout still reports its status") + require.Equal(t, portForwardStatusDialFailed, status[0]) +} + +func TestPortForwardProtocolMatchesHost(t *testing.T) { + require.Equal(t, pb.PortForwardVsockPort, portForwardVsockPort) + require.Equal(t, pb.PortForwardStatusOK, portForwardStatusOK) + require.Equal(t, pb.PortForwardStatusDialFailed, portForwardStatusDialFailed) +} + +type flakyListener struct { + net.Listener + failures int +} + +func (l *flakyListener) Accept() (net.Conn, error) { + if l.failures > 0 { + l.failures-- + return nil, errors.New("too many open files") + } + return l.Listener.Accept() +} + +func TestServePortForwardsSurvivesTransientAcceptErrors(t *testing.T) { + echoPort := startLoopbackEcho(t) + inner, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + done := make(chan struct{}) + go func() { + servePortForwards(&flakyListener{Listener: inner, failures: 3}, dialLoopback) + close(done) + }() + + host, err := net.Dial("tcp", inner.Addr().String()) + require.NoError(t, err) + defer host.Close() + require.NoError(t, binary.Write(host, binary.BigEndian, echoPort)) + status := make([]byte, 1) + _, err = io.ReadFull(host, status) + require.NoError(t, err) + require.Equal(t, portForwardStatusOK, status[0], "forwarding works after transient accept errors") + + require.NoError(t, inner.Close()) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("serve did not return after the listener closed") + } +} + +func TestPortForwardListenRetriesUntilBound(t *testing.T) { + var attempts int + want, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { want.Close() }) + got := listenWithRetry(func() (net.Listener, error) { + attempts++ + if attempts <= 12 { + return nil, errors.New("vsock not ready") + } + return want, nil + }, time.Microsecond, time.Millisecond) + require.Same(t, want, got) + require.Equal(t, 13, attempts, "listening continues past the old ten-attempt limit") +} diff --git a/lib/system/guest_agent/service_test.go b/lib/system/guest_agent/service_test.go index 7c9bf4215..e86319c98 100644 --- a/lib/system/guest_agent/service_test.go +++ b/lib/system/guest_agent/service_test.go @@ -301,4 +301,9 @@ func TestGuestServiceReportsAgentInfo(t *testing.T) { require.NoError(t, err) require.EqualValues(t, agentProtocolVersion, info.ProtocolVersion) require.Equal(t, agentVersion, info.Version) + if portForwardSupported { + require.Contains(t, info.Capabilities, pb.AgentCapability_AGENT_CAPABILITY_PORT_FORWARD) + } else { + require.Empty(t, info.Capabilities, "Linux agents forward no ports") + } }