Skip to content

Commit a18ab32

Browse files
chruffinschruffins
committed
macOS guests 5/6: guest port forwarding
Forward HTTP and WebSocket requests from /instances/{id}/ports/{port}/* to a TCP port on a macOS guest's loopback interface, through the guest agent on vsock 2223. Hypeman does not start or supervise services in the guest; the image does.
1 parent 782ccd8 commit a18ab32

12 files changed

Lines changed: 902 additions & 16 deletions

File tree

‎cmd/api/api/port_forward.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"net"
8+
"net/http"
9+
"net/http/httputil"
10+
"net/url"
11+
"strconv"
12+
13+
"github.com/go-chi/chi/v5"
14+
"github.com/kernel/hypeman/lib/guest"
15+
"github.com/kernel/hypeman/lib/instances"
16+
"github.com/kernel/hypeman/lib/logger"
17+
mw "github.com/kernel/hypeman/lib/middleware"
18+
)
19+
20+
// PortForwardHandler forwards HTTP and WebSocket requests on
21+
// /instances/{id}/ports/{port}/* to that TCP port on a macOS guest's loopback
22+
// interface. The Authorization header, hypeman's only credential, is not
23+
// forwarded. Responses come back unchanged; rewriting guest URLs is the caller's job.
24+
// There is no response timeout, so long polls work; a client disconnect ends the forward.
25+
func (s *ApiService) PortForwardHandler(w http.ResponseWriter, r *http.Request) {
26+
inst := mw.GetResolvedInstance[instances.Instance](r.Context())
27+
if inst == nil {
28+
http.Error(w, `{"code":"internal_error","message":"resource not resolved"}`, http.StatusInternalServerError)
29+
return
30+
}
31+
if reason := inst.GuestOperationUnsupportedReason(instances.GuestOpPortForward); reason != "" {
32+
writeGuestUnsupported(w, reason)
33+
return
34+
}
35+
port, err := strconv.ParseUint(chi.URLParam(r, "port"), 10, 16)
36+
if err != nil || port == 0 {
37+
http.Error(w, `{"code":"invalid_port","message":"port must be between 1 and 65535"}`, http.StatusBadRequest)
38+
return
39+
}
40+
if inst.State != instances.StateRunning {
41+
http.Error(w, fmt.Sprintf(`{"code":"invalid_state","message":"instance must be running (current state: %s)"}`, inst.State), http.StatusConflict)
42+
return
43+
}
44+
path, rawPath, err := forwardedPath(r)
45+
if err != nil {
46+
http.Error(w, `{"code":"invalid_path","message":"invalid escaped path"}`, http.StatusBadRequest)
47+
return
48+
}
49+
dialer, err := s.InstanceManager.GetVsockDialer(r.Context(), inst.Id)
50+
if err != nil {
51+
http.Error(w, `{"code":"unavailable","message":"guest agent unavailable"}`, http.StatusServiceUnavailable)
52+
return
53+
}
54+
55+
target := net.JoinHostPort("127.0.0.1", strconv.FormatUint(port, 10))
56+
proxy := &httputil.ReverseProxy{
57+
Rewrite: func(pr *httputil.ProxyRequest) {
58+
pr.Out.URL.Scheme = "http"
59+
pr.Out.URL.Host = target
60+
pr.Out.URL.Path = path
61+
pr.Out.URL.RawPath = rawPath
62+
// Guest services such as Chrome DevTools only accept a loopback Host.
63+
pr.Out.Host = target
64+
pr.Out.Header.Del("Authorization")
65+
},
66+
Transport: &http.Transport{
67+
DisableKeepAlives: true,
68+
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
69+
return guest.DialPort(ctx, dialer, uint16(port))
70+
},
71+
},
72+
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
73+
logger.FromContext(r.Context()).WarnContext(r.Context(), "guest port forward failed", "instance_id", inst.Id, "port", port, "error", err)
74+
if errors.Is(err, guest.ErrGuestPortUnavailable) {
75+
http.Error(w, `{"code":"port_unavailable","message":"nothing is listening on the guest port"}`, http.StatusBadGateway)
76+
return
77+
}
78+
if errors.Is(err, guest.ErrPortForwarderUnavailable) {
79+
http.Error(w, `{"code":"unavailable","message":"guest agent unavailable"}`, http.StatusServiceUnavailable)
80+
return
81+
}
82+
http.Error(w, `{"code":"bad_gateway","message":"guest port forward failed"}`, http.StatusBadGateway)
83+
},
84+
}
85+
proxy.ServeHTTP(w, r)
86+
}
87+
88+
// forwardedPath returns the guest path for the wildcard after /ports/{port}. chi
89+
// matches on the escaped path when the request has one, so the wildcard keeps its
90+
// escaping (for example %2F) and is unescaped here rather than escaped twice.
91+
func forwardedPath(r *http.Request) (path, rawPath string, err error) {
92+
suffix := chi.URLParam(r, "*")
93+
if r.URL.RawPath == "" {
94+
return "/" + suffix, "", nil
95+
}
96+
unescaped, err := url.PathUnescape(suffix)
97+
if err != nil {
98+
return "", "", err
99+
}
100+
return "/" + unescaped, "/" + suffix, nil
101+
}

‎cmd/api/api/port_forward_test.go‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"encoding/binary"
6+
"errors"
7+
"io"
8+
"net"
9+
"net/http"
10+
"net/http/httptest"
11+
"strings"
12+
"testing"
13+
14+
"github.com/go-chi/chi/v5"
15+
"github.com/gorilla/websocket"
16+
"github.com/kernel/hypeman/lib/guest"
17+
"github.com/kernel/hypeman/lib/hypervisor"
18+
"github.com/kernel/hypeman/lib/images"
19+
"github.com/kernel/hypeman/lib/instances"
20+
mw "github.com/kernel/hypeman/lib/middleware"
21+
"github.com/kernel/hypeman/lib/scopes"
22+
"github.com/stretchr/testify/require"
23+
)
24+
25+
// forwardingDialer stands in for the guest agent: it answers the port forward
26+
// handshake and connects every forward to backend, whatever port is named.
27+
type forwardingDialer struct {
28+
backend string
29+
ports chan uint16
30+
dialErr error // the forwarder itself is unreachable
31+
unopened bool // nothing listens on the guest port
32+
}
33+
34+
func (d *forwardingDialer) DialVsock(_ context.Context, port int) (net.Conn, error) {
35+
if port != guest.PortForwardVsockPort {
36+
return nil, io.ErrUnexpectedEOF
37+
}
38+
if d.dialErr != nil {
39+
return nil, d.dialErr
40+
}
41+
host, agent := net.Pipe()
42+
go func() {
43+
defer agent.Close()
44+
var target uint16
45+
if err := binary.Read(agent, binary.BigEndian, &target); err != nil {
46+
return
47+
}
48+
d.ports <- target
49+
if d.unopened {
50+
_, _ = agent.Write([]byte{guest.PortForwardStatusDialFailed})
51+
return
52+
}
53+
conn, err := net.Dial("tcp", d.backend)
54+
if err != nil {
55+
_, _ = agent.Write([]byte{guest.PortForwardStatusDialFailed})
56+
return
57+
}
58+
defer conn.Close()
59+
_, _ = agent.Write([]byte{guest.PortForwardStatusOK})
60+
go func() { _, _ = io.Copy(conn, agent) }()
61+
_, _ = io.Copy(agent, conn)
62+
}()
63+
return host, nil
64+
}
65+
66+
func (d *forwardingDialer) Key() string { return "forwarding" }
67+
68+
type portForwardManager struct {
69+
instances.Manager
70+
dialer hypervisor.VsockDialer
71+
dialerErr error
72+
}
73+
74+
func (m portForwardManager) GetVsockDialer(context.Context, string) (hypervisor.VsockDialer, error) {
75+
return m.dialer, m.dialerErr
76+
}
77+
78+
func newPortForwardServer(t *testing.T, inst *instances.Instance, backend http.Handler, configure ...func(*forwardingDialer, *portForwardManager)) (*httptest.Server, chan uint16) {
79+
t.Helper()
80+
upstream := httptest.NewServer(backend)
81+
t.Cleanup(upstream.Close)
82+
dialer := &forwardingDialer{backend: strings.TrimPrefix(upstream.URL, "http://"), ports: make(chan uint16, 8)}
83+
manager := portForwardManager{dialer: dialer}
84+
for _, c := range configure {
85+
c(dialer, &manager)
86+
}
87+
s := &ApiService{InstanceManager: manager}
88+
r := chi.NewRouter()
89+
r.Use(func(next http.Handler) http.Handler {
90+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
91+
next.ServeHTTP(w, r.WithContext(mw.WithResolvedInstance(r.Context(), inst.Id, inst)))
92+
})
93+
})
94+
for _, method := range scopes.PortForwardMethods {
95+
r.MethodFunc(method, scopes.PortForwardRoute, s.PortForwardHandler)
96+
}
97+
server := httptest.NewServer(r)
98+
t.Cleanup(server.Close)
99+
return server, dialer.ports
100+
}
101+
102+
func runningMacOSInstance() *instances.Instance {
103+
return &instances.Instance{
104+
StoredMetadata: instances.StoredMetadata{Id: "mac", MacOS: &images.MacOSImage{GuestAgent: true}},
105+
State: instances.StateRunning,
106+
}
107+
}
108+
109+
func TestPortForwardForwardsHTTP(t *testing.T) {
110+
var gotPath, gotQuery, gotHost, gotAuth string
111+
server, ports := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
112+
gotPath, gotQuery, gotHost, gotAuth = r.URL.Path, r.URL.RawQuery, r.Host, r.Header.Get("Authorization")
113+
_, _ = io.WriteString(w, `{"Browser":"Chrome"}`)
114+
}))
115+
116+
req, err := http.NewRequest(http.MethodGet, server.URL+"/instances/mac/ports/9222/json/version?x=1", nil)
117+
require.NoError(t, err)
118+
req.Header.Set("Authorization", "Bearer hypeman-token")
119+
resp, err := http.DefaultClient.Do(req)
120+
require.NoError(t, err)
121+
defer resp.Body.Close()
122+
body, err := io.ReadAll(resp.Body)
123+
require.NoError(t, err)
124+
125+
require.Equal(t, http.StatusOK, resp.StatusCode)
126+
require.Equal(t, `{"Browser":"Chrome"}`, string(body))
127+
require.EqualValues(t, 9222, <-ports)
128+
require.Equal(t, "/json/version", gotPath)
129+
require.Equal(t, "x=1", gotQuery)
130+
require.Equal(t, "127.0.0.1:9222", gotHost, "guest services see a loopback Host")
131+
require.Empty(t, gotAuth, "hypeman credentials must not reach the guest")
132+
}
133+
134+
func TestPortForwardForwardsWebSocket(t *testing.T) {
135+
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
136+
server, _ := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
137+
conn, err := upgrader.Upgrade(w, r, nil)
138+
if err != nil {
139+
return
140+
}
141+
defer conn.Close()
142+
kind, data, err := conn.ReadMessage()
143+
if err != nil {
144+
return
145+
}
146+
_ = conn.WriteMessage(kind, data)
147+
}))
148+
149+
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http")+"/instances/mac/ports/9222/devtools/browser/abc", nil)
150+
require.NoError(t, err)
151+
defer conn.Close()
152+
require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"id":1}`)))
153+
_, data, err := conn.ReadMessage()
154+
require.NoError(t, err)
155+
require.Equal(t, `{"id":1}`, string(data))
156+
}
157+
158+
func TestPortForwardRejections(t *testing.T) {
159+
noAgent := runningMacOSInstance()
160+
noAgent.MacOS.GuestAgent = false
161+
stopped := runningMacOSInstance()
162+
stopped.State = instances.StateStopped
163+
linux := &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "linux"}, State: instances.StateRunning}
164+
165+
for _, tc := range []struct {
166+
name string
167+
inst *instances.Instance
168+
port string
169+
code int
170+
}{
171+
{"linux guest", linux, "9222", http.StatusNotImplemented},
172+
{"macOS without agent", noAgent, "9222", http.StatusNotImplemented},
173+
{"port zero", runningMacOSInstance(), "0", http.StatusBadRequest},
174+
{"port out of range", runningMacOSInstance(), "70000", http.StatusBadRequest},
175+
{"stopped", stopped, "9222", http.StatusConflict},
176+
} {
177+
t.Run(tc.name, func(t *testing.T) {
178+
server, _ := newPortForwardServer(t, tc.inst, http.NotFoundHandler())
179+
resp, err := http.Get(server.URL + "/instances/" + tc.inst.Id + "/ports/" + tc.port + "/json")
180+
require.NoError(t, err)
181+
resp.Body.Close()
182+
require.Equal(t, tc.code, resp.StatusCode)
183+
})
184+
}
185+
}
186+
187+
func TestPortForwardKeepsEscapedPath(t *testing.T) {
188+
var gotRawPath string
189+
server, _ := newPortForwardServer(t, runningMacOSInstance(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
190+
gotRawPath = r.URL.EscapedPath()
191+
}))
192+
resp, err := http.Get(server.URL + "/instances/mac/ports/9222/a%2Fb/c%20d")
193+
require.NoError(t, err)
194+
resp.Body.Close()
195+
require.Equal(t, http.StatusOK, resp.StatusCode)
196+
require.Equal(t, "/a%2Fb/c%20d", gotRawPath, "escapes reach the guest once, not twice")
197+
}
198+
199+
func TestPortForwardForwardFailures(t *testing.T) {
200+
for _, tc := range []struct {
201+
name string
202+
configure func(*forwardingDialer, *portForwardManager)
203+
code int
204+
body string
205+
}{
206+
{"nothing listening", func(d *forwardingDialer, _ *portForwardManager) { d.unopened = true }, http.StatusBadGateway, "port_unavailable"},
207+
{"forwarder unreachable", func(d *forwardingDialer, _ *portForwardManager) { d.dialErr = errors.New("connection refused") }, http.StatusServiceUnavailable, "unavailable"},
208+
{"agent disabled", func(_ *forwardingDialer, m *portForwardManager) { m.dialerErr = instances.ErrInvalidRequest }, http.StatusServiceUnavailable, "unavailable"},
209+
} {
210+
t.Run(tc.name, func(t *testing.T) {
211+
server, _ := newPortForwardServer(t, runningMacOSInstance(), http.NotFoundHandler(), tc.configure)
212+
resp, err := http.Get(server.URL + "/instances/mac/ports/9222/json")
213+
require.NoError(t, err)
214+
body, err := io.ReadAll(resp.Body)
215+
resp.Body.Close()
216+
require.NoError(t, err)
217+
require.Equal(t, tc.code, resp.StatusCode)
218+
require.Contains(t, string(body), tc.body)
219+
})
220+
}
221+
}

‎cmd/api/main.go‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -447,9 +447,10 @@ func run() error {
447447
// See: https://github.com/oapi-codegen/nethttp-middleware#usage
448448
spec.Servers = nil
449449

450-
// Custom exec endpoint (outside OpenAPI spec, uses WebSocket)
450+
// Exec and cp (WebSocket) and port forwarding (HTTP and WebSocket proxy) are
451+
// outside the OpenAPI spec and share one middleware chain.
451452
// Note: No otelchi here as WebSocket doesn't work well with tracing middleware
452-
r.With(
453+
guestRoutes := r.With(
453454
middleware.RequestID,
454455
middleware.RealIP,
455456
middleware.Recoverer,
@@ -458,19 +459,12 @@ func run() error {
458459
mw.JwtAuth(app.Config.JwtSecret),
459460
scopes.RequireScope(scopes.InstanceWrite),
460461
mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder),
461-
).Get("/instances/{id}/exec", app.ApiService.ExecHandler)
462-
463-
// Custom cp endpoint (outside OpenAPI spec, uses WebSocket)
464-
r.With(
465-
middleware.RequestID,
466-
middleware.RealIP,
467-
middleware.Recoverer,
468-
mw.InjectLogger(logger),
469-
mw.AccessLogger(accessLogger),
470-
mw.JwtAuth(app.Config.JwtSecret),
471-
scopes.RequireScope(scopes.InstanceWrite),
472-
mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder),
473-
).Get("/instances/{id}/cp", app.ApiService.CpHandler)
462+
)
463+
guestRoutes.Get("/instances/{id}/exec", app.ApiService.ExecHandler)
464+
guestRoutes.Get("/instances/{id}/cp", app.ApiService.CpHandler)
465+
for _, method := range scopes.PortForwardMethods {
466+
guestRoutes.MethodFunc(method, scopes.PortForwardRoute, app.ApiService.PortForwardHandler)
467+
}
474468

475469
// Create builder VM resolver for secure token authentication
476470
// This validates that token requests from builder VMs are for their authorized repos only

0 commit comments

Comments
 (0)