debug: replace RtlCaptureStackBackTrace (which was spuriously failing) with a new implementation which uses RtlVirtualUnwind instead (#12740)

windows: add RtlCaptureContext, RtlLookupFunctionEntry, RtlVirtualUnwind and supporting types
windows: fix alignment of CONTEXT structs to match winnt.h as required by RtlCaptureContext (fxsave instr)
windows aarch64: fix __chkstk being defined twice if libc is not linked on msvc

Co-authored-by: Jakub Konka <kubkon@jakubkonka.com>

Author: kcbanner

lib/std/debug.zig

@@ -206,17 +206,12 @@ pub fn captureStackTrace(first_address: ?usize, stack_trace: *std.builtin.StackT
     if (native_os == .windows) {
         const addrs = stack_trace.instruction_addresses;
         const first_addr = first_address orelse {
-            stack_trace.index = windows.ntdll.RtlCaptureStackBackTrace(
-                0,
-                @intCast(u32, addrs.len),
-                @ptrCast(**anyopaque, addrs.ptr),
-                null,
-            );
+            stack_trace.index = walkStackWindows(addrs[0..]);
             return;
         };
         var addr_buf_stack: [32]usize = undefined;
         const addr_buf = if (addr_buf_stack.len > addrs.len) addr_buf_stack[0..] else addrs;
-        const n = windows.ntdll.RtlCaptureStackBackTrace(0, @intCast(u32, addr_buf.len), @ptrCast(**anyopaque, addr_buf.ptr), null);
+        const n = walkStackWindows(addr_buf[0..]);
         const first_index = for (addr_buf[0..n]) |addr, i| {
             if (addr == first_addr) {
                 break i;
@@ -573,14 +568,56 @@ pub fn writeCurrentStackTrace(
     }
 }
 
+pub noinline fn walkStackWindows(addresses: []usize) usize {
+    if (builtin.cpu.arch == .x86) {
+        // RtlVirtualUnwind doesn't exist on x86
+        return windows.ntdll.RtlCaptureStackBackTrace(0, addresses.len, @ptrCast(**anyopaque, addresses.ptr), null);
+    }
+
+    const tib = @ptrCast(*const windows.NT_TIB, &windows.teb().Reserved1);
+
+    var context: windows.CONTEXT = std.mem.zeroes(windows.CONTEXT);
+    windows.ntdll.RtlCaptureContext(&context);
+
+    var i: usize = 0;
+    var image_base: usize = undefined;
+    var history_table: windows.UNWIND_HISTORY_TABLE = std.mem.zeroes(windows.UNWIND_HISTORY_TABLE);
+
+    while (i < addresses.len) : (i += 1) {
+        const current_regs = context.getRegs();
+        if (windows.ntdll.RtlLookupFunctionEntry(current_regs.ip, &image_base, &history_table)) |runtime_function| {
+            var handler_data: ?*anyopaque = null;
+            var establisher_frame: u64 = undefined;
+            _ = windows.ntdll.RtlVirtualUnwind(windows.UNW_FLAG_NHANDLER, image_base, current_regs.ip, runtime_function, &context, &handler_data, &establisher_frame, null);
+        } else {
+            // leaf function
+            context.setIp(@intToPtr(*u64, current_regs.sp).*);
+            context.setSp(current_regs.sp + @sizeOf(usize));
+        }
+
+        const next_regs = context.getRegs();
+        if (next_regs.sp < @ptrToInt(tib.StackLimit) or next_regs.sp > @ptrToInt(tib.StackBase)) {
+            break;
+        }
+
+        if (next_regs.ip == 0) {
+            break;
+        }
+
+        addresses[i] = next_regs.ip;
+    }
+
+    return i;
+}
+
 pub fn writeCurrentStackTraceWindows(
     out_stream: anytype,
     debug_info: *DebugInfo,
     tty_config: TTY.Config,
     start_addr: ?usize,
 ) !void {
     var addr_buf: [1024]usize = undefined;
-    const n = windows.ntdll.RtlCaptureStackBackTrace(0, addr_buf.len, @ptrCast(**anyopaque, &addr_buf), null);
+    const n = walkStackWindows(addr_buf[0..]);
     const addrs = addr_buf[0..n];
     var start_i: usize = if (start_addr) |saddr| blk: {
         for (addrs) |addr, i| {

lib/std/os/windows.zig

@@ -31,6 +31,8 @@ pub const winmm = @import("windows/winmm.zig");
 
 pub const self_process_handle = @intToPtr(HANDLE, maxInt(usize));
 
+const Self = @This();
+
 pub const OpenError = error{
     IsDir,
     NotDir,
@@ -3282,7 +3284,7 @@ pub usingnamespace switch (native_arch) {
         };
 
         pub const CONTEXT = extern struct {
-            P1Home: DWORD64,
+            P1Home: DWORD64 align(16),
             P2Home: DWORD64,
             P3Home: DWORD64,
             P4Home: DWORD64,
@@ -3352,9 +3354,28 @@ pub usingnamespace switch (native_arch) {
             LastExceptionToRip: DWORD64,
             LastExceptionFromRip: DWORD64,
 
-            pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize } {
-                return .{ .bp = ctx.Rbp, .ip = ctx.Rip };
+            pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {
+                return .{ .bp = ctx.Rbp, .ip = ctx.Rip, .sp = ctx.Rsp };
+            }
+
+            pub fn setIp(ctx: *CONTEXT, ip: usize) void {
+                ctx.Rip = ip;
             }
+
+            pub fn setSp(ctx: *CONTEXT, sp: usize) void {
+                ctx.Rsp = sp;
+            }
+        };
+
+        pub const RUNTIME_FUNCTION = extern struct {
+            BeginAddress: DWORD,
+            EndAddress: DWORD,
+            UnwindData: DWORD,
+        };
+
+        pub const KNONVOLATILE_CONTEXT_POINTERS = extern struct {
+            FloatingContext: [16]?*M128A,
+            IntegerContext: [16]?*ULONG64,
         };
     },
     .aarch64 => struct {
@@ -3370,7 +3391,7 @@ pub usingnamespace switch (native_arch) {
         };
 
         pub const CONTEXT = extern struct {
-            ContextFlags: ULONG,
+            ContextFlags: ULONG align(16),
             Cpsr: ULONG,
             DUMMYUNIONNAME: extern union {
                 DUMMYSTRUCTNAME: extern struct {
@@ -3418,12 +3439,60 @@ pub usingnamespace switch (native_arch) {
             Wcr: [2]DWORD,
             Wvr: [2]DWORD64,
 
-            pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize } {
+            pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {
                 return .{
                     .bp = ctx.DUMMYUNIONNAME.DUMMYSTRUCTNAME.Fp,
                     .ip = ctx.Pc,
+                    .sp = ctx.Sp,
                 };
             }
+
+            pub fn setIp(ctx: *CONTEXT, ip: usize) void {
+                ctx.Pc = ip;
+            }
+
+            pub fn setSp(ctx: *CONTEXT, sp: usize) void {
+                ctx.Sp = sp;
+            }
+        };
+
+        pub const RUNTIME_FUNCTION = extern struct {
+            BeginAddress: DWORD,
+            DUMMYUNIONNAME: extern union {
+                UnwindData: DWORD,
+                DUMMYSTRUCTNAME: packed struct {
+                    Flag: u2,
+                    FunctionLength: u11,
+                    RegF: u3,
+                    RegI: u4,
+                    H: u1,
+                    CR: u2,
+                    FrameSize: u9,
+                },
+            },
+        };
+
+        pub const KNONVOLATILE_CONTEXT_POINTERS = extern struct {
+            X19: ?*DWORD64,
+            X20: ?*DWORD64,
+            X21: ?*DWORD64,
+            X22: ?*DWORD64,
+            X23: ?*DWORD64,
+            X24: ?*DWORD64,
+            X25: ?*DWORD64,
+            X26: ?*DWORD64,
+            X27: ?*DWORD64,
+            X28: ?*DWORD64,
+            Fp: ?*DWORD64,
+            Lr: ?*DWORD64,
+            D8: ?*DWORD64,
+            D9: ?*DWORD64,
+            D10: ?*DWORD64,
+            D11: ?*DWORD64,
+            D12: ?*DWORD64,
+            D13: ?*DWORD64,
+            D14: ?*DWORD64,
+            D15: ?*DWORD64,
         };
     },
     else => struct {},
@@ -3436,6 +3505,36 @@ pub const EXCEPTION_POINTERS = extern struct {
 
 pub const VECTORED_EXCEPTION_HANDLER = *const fn (ExceptionInfo: *EXCEPTION_POINTERS) callconv(WINAPI) c_long;
 
+pub const EXCEPTION_DISPOSITION = i32;
+pub const EXCEPTION_ROUTINE = *const fn (
+    ExceptionRecord: ?*EXCEPTION_RECORD,
+    EstablisherFrame: PVOID,
+    ContextRecord: *(Self.CONTEXT),
+    DispatcherContext: PVOID,
+) callconv(WINAPI) EXCEPTION_DISPOSITION;
+
+pub const UNWIND_HISTORY_TABLE_SIZE = 12;
+pub const UNWIND_HISTORY_TABLE_ENTRY = extern struct {
+    ImageBase: ULONG64,
+    FunctionEntry: *Self.RUNTIME_FUNCTION,
+};
+
+pub const UNWIND_HISTORY_TABLE = extern struct {
+    Count: ULONG,
+    LocalHint: BYTE,
+    GlobalHint: BYTE,
+    Search: BYTE,
+    Once: BYTE,
+    LowAddress: ULONG64,
+    HighAddress: ULONG64,
+    Entry: [UNWIND_HISTORY_TABLE_SIZE]UNWIND_HISTORY_TABLE_ENTRY,
+};
+
+pub const UNW_FLAG_NHANDLER = 0x0;
+pub const UNW_FLAG_EHANDLER = 0x1;
+pub const UNW_FLAG_UHANDLER = 0x2;
+pub const UNW_FLAG_CHAININFO = 0x4;
+
 pub const OBJECT_ATTRIBUTES = extern struct {
     Length: ULONG,
     RootDirectory: ?HANDLE,
@@ -3494,6 +3593,21 @@ pub const TEB = extern struct {
     TlsExpansionSlots: PVOID,
 };
 
+pub const EXCEPTION_REGISTRATION_RECORD = extern struct {
+    Next: ?*EXCEPTION_REGISTRATION_RECORD,
+    Handler: ?*EXCEPTION_DISPOSITION,
+};
+
+pub const NT_TIB = extern struct {
+    ExceptionList: ?*EXCEPTION_REGISTRATION_RECORD,
+    StackBase: PVOID,
+    StackLimit: PVOID,
+    SubSystemTib: PVOID,
+    DUMMYUNIONNAME: extern union { FiberData: PVOID, Version: DWORD },
+    ArbitraryUserPointer: PVOID,
+    Self: ?*@This(),
+};
+
 /// Process Environment Block
 /// Microsoft documentation of this is incomplete, the fields here are taken from various resources including:
 ///  - https://github.com/wine-mirror/wine/blob/1aff1e6a370ee8c0213a0fd4b220d121da8527aa/include/winternl.h#L269

lib/std/os/windows/kernel32.zig

@@ -5,8 +5,10 @@ const BOOL = windows.BOOL;
 const BOOLEAN = windows.BOOLEAN;
 const CONDITION_VARIABLE = windows.CONDITION_VARIABLE;
 const CONSOLE_SCREEN_BUFFER_INFO = windows.CONSOLE_SCREEN_BUFFER_INFO;
+const CONTEXT = windows.CONTEXT;
 const COORD = windows.COORD;
 const DWORD = windows.DWORD;
+const DWORD64 = windows.DWORD64;
 const FILE_INFO_BY_HANDLE_CLASS = windows.FILE_INFO_BY_HANDLE_CLASS;
 const HANDLE = windows.HANDLE;
 const HMODULE = windows.HMODULE;
@@ -60,6 +62,10 @@ const INIT_ONCE_FN = windows.INIT_ONCE_FN;
 const PMEMORY_BASIC_INFORMATION = windows.PMEMORY_BASIC_INFORMATION;
 const REGSAM = windows.REGSAM;
 const LSTATUS = windows.LSTATUS;
+const UNWIND_HISTORY_TABLE = windows.UNWIND_HISTORY_TABLE;
+const RUNTIME_FUNCTION = windows.RUNTIME_FUNCTION;
+const KNONVOLATILE_CONTEXT_POINTERS = windows.KNONVOLATILE_CONTEXT_POINTERS;
+const EXCEPTION_ROUTINE = windows.EXCEPTION_ROUTINE;
 
 pub extern "kernel32" fn AddVectoredExceptionHandler(First: c_ulong, Handler: ?VECTORED_EXCEPTION_HANDLER) callconv(WINAPI) ?*anyopaque;
 pub extern "kernel32" fn RemoveVectoredExceptionHandler(Handle: HANDLE) callconv(WINAPI) c_ulong;
@@ -292,6 +298,25 @@ pub extern "kernel32" fn ReadFile(
 
 pub extern "kernel32" fn RemoveDirectoryW(lpPathName: [*:0]const u16) callconv(WINAPI) BOOL;
 
+pub extern "kernel32" fn RtlCaptureContext(ContextRecord: *CONTEXT) callconv(WINAPI) void;
+
+pub extern "kernel32" fn RtlLookupFunctionEntry(
+    ControlPc: DWORD64,
+    ImageBase: *DWORD64,
+    HistoryTable: *UNWIND_HISTORY_TABLE,
+) callconv(WINAPI) ?*RUNTIME_FUNCTION;
+
+pub extern "kernel32" fn RtlVirtualUnwind(
+    HandlerType: DWORD,
+    ImageBase: DWORD64,
+    ControlPc: DWORD64,
+    FunctionEntry: *RUNTIME_FUNCTION,
+    ContextRecord: *CONTEXT,
+    HandlerData: *?PVOID,
+    EstablisherFrame: *DWORD64,
+    ContextPointers: ?*KNONVOLATILE_CONTEXT_POINTERS,
+) callconv(WINAPI) *EXCEPTION_ROUTINE;
+
 pub extern "kernel32" fn SetConsoleTextAttribute(hConsoleOutput: HANDLE, wAttributes: WORD) callconv(WINAPI) BOOL;
 
 pub extern "kernel32" fn SetConsoleCtrlHandler(

lib/std/os/windows/ntdll.zig

@@ -3,6 +3,7 @@ const windows = std.os.windows;
 
 const BOOL = windows.BOOL;
 const DWORD = windows.DWORD;
+const DWORD64 = windows.DWORD64;
 const ULONG = windows.ULONG;
 const WINAPI = windows.WINAPI;
 const NTSTATUS = windows.NTSTATUS;
@@ -24,6 +25,11 @@ const SIZE_T = windows.SIZE_T;
 const CURDIR = windows.CURDIR;
 const PCWSTR = windows.PCWSTR;
 const RTL_QUERY_REGISTRY_TABLE = windows.RTL_QUERY_REGISTRY_TABLE;
+const CONTEXT = windows.CONTEXT;
+const UNWIND_HISTORY_TABLE = windows.UNWIND_HISTORY_TABLE;
+const RUNTIME_FUNCTION = windows.RUNTIME_FUNCTION;
+const KNONVOLATILE_CONTEXT_POINTERS = windows.KNONVOLATILE_CONTEXT_POINTERS;
+const EXCEPTION_ROUTINE = windows.EXCEPTION_ROUTINE;
 
 pub const THREADINFOCLASS = enum(c_int) {
     ThreadBasicInformation,
@@ -99,6 +105,22 @@ pub extern "ntdll" fn RtlCaptureStackBackTrace(
     BackTrace: **anyopaque,
     BackTraceHash: ?*DWORD,
 ) callconv(WINAPI) WORD;
+pub extern "ntdll" fn RtlCaptureContext(ContextRecord: *CONTEXT) callconv(WINAPI) void;
+pub extern "ntdll" fn RtlLookupFunctionEntry(
+    ControlPc: DWORD64,
+    ImageBase: *DWORD64,
+    HistoryTable: *UNWIND_HISTORY_TABLE,
+) callconv(WINAPI) ?*RUNTIME_FUNCTION;
+pub extern "ntdll" fn RtlVirtualUnwind(
+    HandlerType: DWORD,
+    ImageBase: DWORD64,
+    ControlPc: DWORD64,
+    FunctionEntry: *RUNTIME_FUNCTION,
+    ContextRecord: *CONTEXT,
+    HandlerData: *?PVOID,
+    EstablisherFrame: *DWORD64,
+    ContextPointers: ?*KNONVOLATILE_CONTEXT_POINTERS,
+) callconv(WINAPI) *EXCEPTION_ROUTINE;
 pub extern "ntdll" fn NtQueryInformationFile(
     FileHandle: HANDLE,
     IoStatusBlock: *IO_STATUS_BLOCK,