Fix the location of the thread metadata on riscv64 (#96)

On riscv64, the thread pointer doesn't point to the first ABI-exposed
field in the thread-local storage. Add an empty `thread_pointee` field
to represent the place the thread pointer points to, and fix the
address calculation code to point there.

And add a tls testcase that creates a `#[thread_local]` variable and
checks that it's initialized and can be accessed properly.

Author: sunfishcode

src/thread/linux_raw.rs

@@ -114,45 +114,59 @@ struct Metadata {
     thread: ThreadData,
 }
 
-/// Fields which accessed by user code via known offsets from the platform
-/// thread-pointer register.
+/// Fields which accessed by user code via well-known offsets from the platform
+/// thread-pointer register. Specifically, the thread-pointer register points
+/// to the `thread_pointee` field.
 #[repr(C)]
 #[cfg_attr(target_arch = "arm", repr(align(8)))]
 struct Abi {
+    /// The address the thread pointer points to.
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    thread_pointee: [u8; 0],
+
     /// The ABI-exposed `canary` field.
     #[cfg(any(target_arch = "aarch64", target_arch = "arm", target_arch = "riscv64"))]
     canary: usize,
 
+    /// The address the thread pointer points to.
+    #[cfg(any(target_arch = "aarch64", target_arch = "arm",))]
+    thread_pointee: [u8; 0],
+
     /// The ABI-exposed `dtv` field (though we don't yet implement dynamic
     /// linking).
     #[cfg(any(target_arch = "aarch64", target_arch = "arm", target_arch = "riscv64"))]
     dtv: *const c_void,
 
+    /// The address the thread pointer points to.
+    #[cfg(target_arch = "riscv64")]
+    thread_pointee: [u8; 0],
+
+    /// Padding to put the TLS data which follows at its well-known offset.
+    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
+    _pad: [usize; 1],
+
+    /// Padding to put the TLS data which follows at its well-known offset.
+    #[cfg(target_arch = "riscv64")]
+    _pad: [usize; 0],
+
     /// x86 and x86-64 put a copy of the thread-pointer register at the memory
     /// location pointed to by the thread-pointer register, because reading the
     /// thread-pointer register directly is slow.
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    this: *mut Abi,
+    this: *mut c_void,
 
     /// The ABI-exposed `dtv` field (though we don't yet implement dynamic
     /// linking).
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     dtv: *const c_void,
 
+    /// Padding to put the `canary` field at its well-known offset.
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     _pad: [usize; 3],
 
     /// The ABI-exposed `canary` field.
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     canary: usize,
-
-    /// Padding to put the TLS data which follows at its known offset.
-    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
-    _pad: [usize; 1],
-
-    /// Padding to put the TLS data which follows at its known offset.
-    #[cfg(target_arch = "riscv64")]
-    _pad: [usize; 0],
 }
 
 /// Information obtained from the `DT_TLS` segment of the executable.
@@ -270,7 +284,9 @@ pub use rustix::thread::Pid as ThreadId;
 ///
 /// This function is similar to `create_thread` except that the OS thread is
 /// already created, and already has a stack (which we need to locate), and is
-/// already running.
+/// already running. We still need to create the thread [`Metadata`], copy in
+/// the TLS initializers, and point the thread pointer to it so that it follows
+/// the thread ABI that all the other threads follow.
 ///
 /// # Safety
 ///
@@ -346,7 +362,7 @@ pub(super) unsafe fn initialize_main_thread(mem: *mut c_void) {
 
     let tls_data = new.add(tls_data_bottom);
     let metadata: *mut Metadata = new.add(header).cast();
-    let newtls: *mut Abi = &mut (*metadata).abi;
+    let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast();
 
     let thread_id_ptr = (*metadata).thread.thread_id.as_ptr();
     let tid = rustix::runtime::set_tid_address(thread_id_ptr.cast());
@@ -364,6 +380,7 @@ pub(super) unsafe fn initialize_main_thread(mem: *mut c_void) {
             #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
             this: newtls,
             _pad: Default::default(),
+            thread_pointee: [],
         },
         thread: ThreadData::new(
             Some(tid),
@@ -390,7 +407,7 @@ pub(super) unsafe fn initialize_main_thread(mem: *mut c_void) {
     .fill(0);
 
     // Point the platform thread-pointer register at the new thread metadata.
-    set_thread_pointer(newtls.cast::<u8>().cast());
+    set_thread_pointer(newtls);
 }
 
 /// Creates a new thread.
@@ -486,7 +503,7 @@ pub unsafe fn create_thread(
 
         let tls_data = map.add(tls_data_bottom);
         let metadata: *mut Metadata = map.add(header).cast();
-        let newtls: *mut Abi = &mut (*metadata).abi;
+        let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast();
 
         // Copy the current thread's canary to the new thread.
         let canary = (*current_metadata()).abi.canary;
@@ -499,6 +516,7 @@ pub unsafe fn create_thread(
                 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
                 this: newtls,
                 _pad: Default::default(),
+                thread_pointee: [],
             },
             thread: ThreadData::new(
                 None, // the real tid will be written by `clone`.
@@ -560,7 +578,7 @@ pub unsafe fn create_thread(
             stack.cast(),
             thread_id_ptr,
             thread_id_ptr,
-            newtls.cast::<u8>().cast(),
+            newtls,
             core::mem::transmute(fn_),
             args.len(),
         );
@@ -909,7 +927,7 @@ pub fn at_thread_exit(func: Box<dyn FnOnce()>) {
 fn current_metadata() -> *mut Metadata {
     thread_pointer()
         .cast::<u8>()
-        .wrapping_sub(offset_of!(Metadata, abi))
+        .wrapping_sub(offset_of!(Metadata, abi) + offset_of!(Abi, thread_pointee))
         .cast()
 }
 
@@ -976,8 +994,8 @@ pub fn current_thread_tls_addr(offset: usize) -> *mut c_void {
     {
         thread_pointer()
             .cast::<u8>()
+            .wrapping_add(size_of::<Abi>() - offset_of!(Abi, thread_pointee))
             .wrapping_add(TLS_OFFSET)
-            .wrapping_add(size_of::<Abi>())
             .wrapping_add(offset)
             .cast()
     }
@@ -990,8 +1008,8 @@ pub fn current_thread_tls_addr(offset: usize) -> *mut c_void {
     unsafe {
         thread_pointer()
             .cast::<u8>()
-            .wrapping_add(TLS_OFFSET)
             .wrapping_sub(STARTUP_TLS_INFO.mem_size)
+            .wrapping_add(TLS_OFFSET)
             .wrapping_add(offset)
             .cast()
     }

test-crates/tls/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "tls"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+origin = { path = "../..", default-features = false, features = ["origin-thread", "origin-start"] }
+atomic-dbg = { version = "0.1.8", default-features = false }
+rustix-dlmalloc = { version = "0.1.0", features = ["global"] }
+compiler_builtins = { version = "0.1.101", features = ["mem"] }
+
+# This is just a test crate, and not part of the origin workspace.
+[workspace]

test-crates/tls/README.md

@@ -0,0 +1,3 @@
+This is similar to the `origin-start` example, but adds `#[thread_local]`
+variables and a bunch of asserts to ensure that they're initialized and
+mutated properly.

test-crates/tls/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo:rustc-link-arg=-nostartfiles");
+}

test-crates/tls/src/main.rs

@@ -0,0 +1,128 @@
+//! Like `origin-start`, but add a `#[thread_local]` variable and asserts
+//! to ensure that it works.
+
+#![no_std]
+#![no_main]
+#![allow(internal_features)]
+#![feature(lang_items)]
+#![feature(core_intrinsics)]
+#![feature(thread_local)]
+#![feature(strict_provenance)]
+#![feature(const_mut_refs)]
+
+extern crate alloc;
+extern crate compiler_builtins;
+
+use alloc::boxed::Box;
+use atomic_dbg::dbg;
+use core::arch::asm;
+use core::ptr::{addr_of_mut, invalid_mut};
+use origin::program::*;
+use origin::thread::*;
+
+#[panic_handler]
+fn panic(panic: &core::panic::PanicInfo<'_>) -> ! {
+    dbg!(panic);
+    core::intrinsics::abort();
+}
+
+#[lang = "eh_personality"]
+extern "C" fn eh_personality() {}
+
+#[global_allocator]
+static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc;
+
+#[no_mangle]
+unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 {
+    // Assert that the main thread initialized its TLS properly.
+    check_eq(TEST_DATA.0);
+
+    // Mutate one of the TLS fields.
+    THREAD_LOCAL[1] = invalid_mut(77);
+
+    // Assert that the mutation happened properly.
+    check_eq([TEST_DATA.0[0], invalid_mut(77), TEST_DATA.0[2]]);
+
+    at_exit(Box::new(|| {
+        // This is the last thing to run. Assert that we see the value stored
+        // by the `at_thread_exit` callback.
+        check_eq([TEST_DATA.0[0], invalid_mut(79), TEST_DATA.0[2]]);
+
+        // Mutate one of the TLS fields.
+        THREAD_LOCAL[1] = invalid_mut(80);
+    }));
+    at_thread_exit(Box::new(|| {
+        // Assert that we see the value stored at the end of `main`.
+        check_eq([TEST_DATA.0[0], invalid_mut(78), TEST_DATA.0[2]]);
+
+        // Mutate one of the TLS fields.
+        THREAD_LOCAL[1] = invalid_mut(79);
+    }));
+
+    let thread = create_thread(
+        |_args| {
+            // Assert that the new thread initialized its TLS properly.
+            check_eq(TEST_DATA.0);
+
+            // Mutate one of the TLS fields.
+            THREAD_LOCAL[1] = invalid_mut(175);
+
+            // Assert that the mutation happened properly.
+            check_eq([TEST_DATA.0[0], invalid_mut(175), TEST_DATA.0[2]]);
+
+            at_thread_exit(Box::new(|| {
+                // Assert that we still see the value stored in the thread.
+                check_eq([TEST_DATA.0[0], invalid_mut(175), TEST_DATA.0[2]]);
+            }));
+
+            None
+        },
+        &[],
+        default_stack_size(),
+        default_guard_size(),
+    )
+    .unwrap();
+
+    join_thread(thread);
+
+    // Assert that the main thread's TLS is still in place.
+    check_eq([TEST_DATA.0[0], invalid_mut(77), TEST_DATA.0[2]]);
+
+    // Mutate one of the TLS fields.
+    THREAD_LOCAL[1] = invalid_mut(78);
+
+    // Assert that the mutation happened properly.
+    check_eq([TEST_DATA.0[0], invalid_mut(78), TEST_DATA.0[2]]);
+
+    exit(200);
+}
+
+struct SyncTestData([*const u32; 3]);
+unsafe impl Sync for SyncTestData {}
+static TEST_DATA: SyncTestData = unsafe {
+    SyncTestData([
+        invalid_mut(0xa0b1a2b3a4b5a6b7_u64 as usize),
+        addr_of_mut!(SOME_REGULAR_DATA),
+        addr_of_mut!(SOME_ZERO_DATA),
+    ])
+};
+
+#[thread_local]
+static mut THREAD_LOCAL: [*const u32; 3] = TEST_DATA.0;
+
+// Some variables to point to.
+static mut SOME_REGULAR_DATA: u32 = 909;
+static mut SOME_ZERO_DATA: u32 = 0;
+
+fn check_eq(data: [*const u32; 3]) {
+    unsafe {
+        // Check `THREAD_LOCAL` using a static address.
+        asm!("# {}", in(reg) THREAD_LOCAL.as_mut_ptr(), options(nostack, preserves_flags));
+        assert_eq!(THREAD_LOCAL, data);
+
+        // Check `THREAD_LOCAL` using a dynamic address.
+        let mut thread_local_addr: *mut [*const u32; 3] = &mut THREAD_LOCAL;
+        asm!("# {}", inout(reg) thread_local_addr, options(pure, nomem, nostack, preserves_flags));
+        assert_eq!(*thread_local_addr, data);
+    }
+}

tests/example_crates.rs

@@ -3,6 +3,10 @@
 
 #![feature(cfg_target_abi)]
 
+mod utils;
+
+use utils::{COMMON_STDERR, NO_ALLOC_STDERR};
+
 fn test_crate(
     name: &str,
     args: &[&str],
@@ -11,51 +15,9 @@ fn test_crate(
     stderr: &'static str,
     code: Option<i32>,
 ) {
-    use assert_cmd::Command;
-
-    #[cfg(target_arch = "x86_64")]
-    let arch = "x86_64";
-    #[cfg(target_arch = "aarch64")]
-    let arch = "aarch64";
-    #[cfg(target_arch = "riscv64")]
-    let arch = "riscv64gc";
-    #[cfg(target_arch = "x86")]
-    let arch = "i686";
-    #[cfg(target_arch = "arm")]
-    let arch = "armv5te";
-    #[cfg(target_env = "gnueabi")]
-    let env = "gnueabi";
-    #[cfg(all(target_env = "gnu", target_abi = "eabi"))]
-    let env = "gnueabi";
-    #[cfg(all(target_env = "gnu", not(target_abi = "eabi")))]
-    let env = "gnu";
-
-    let mut command = Command::new("cargo");
-    command.arg("run").arg("--quiet");
-    command.arg(&format!("--target={arch}-unknown-linux-{env}"));
-    command.args(args);
-    command.envs(envs.iter().copied());
-    command.current_dir(format!("example-crates/{name}"));
-    let assert = command.assert();
-    let assert = assert.stdout(stdout).stderr(stderr);
-    if let Some(code) = code {
-        assert.code(code);
-    } else {
-        assert.success();
-    }
+    utils::test_crate("example", name, args, envs, stdout, stderr, code);
 }
 
-/// Stderr output for most of the example crates.
-const COMMON_STDERR: &str = "Hello from main thread\n\
-    Hello from child thread\n\
-    Hello from child thread's at_thread_exit handler\n\
-    Goodbye from main\n\
-    Hello from a main-thread at_thread_exit handler\n\
-    Hello from an at_exit handler\n";
-
-/// Stderr output for the origin-start-no-alloc crate.
-const NO_ALLOC_STDERR: &str = "Hello!\n";
-
 #[test]
 fn example_crate_basic() {
     test_crate("basic", &[], &[], "", COMMON_STDERR, None);