diff --git a/src/asahi/lib/agx_bo.h b/src/asahi/lib/agx_bo.h index d9eed7ac5e1a..f407b01ae650 100644 --- a/src/asahi/lib/agx_bo.h +++ b/src/asahi/lib/agx_bo.h @@ -45,6 +45,13 @@ enum agx_va_flags { /* VA must be fixed, otherwise allocated by the driver. */ AGX_VA_FIXED = (1 << 1), + + /* VA must be inside the capture/replay range. Only buffers created with + * VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT use it, and no other + * allocation draws from it, so a captured address stays free for the + * replay. Not allowed together with AGX_VA_USC. + */ + AGX_VA_CAPTURE = (1 << 2), }; struct agx_va { diff --git a/src/asahi/lib/agx_device.c b/src/asahi/lib/agx_device.c index f158e0e7acea..6bc96b0c6b32 100644 --- a/src/asahi/lib/agx_device.c +++ b/src/asahi/lib/agx_device.c @@ -646,7 +646,7 @@ agx_open_device(void *memctx, struct agx_device *dev) assert(((user_start + (user_size - 1)) & dev->sparse_ro_offset) == 0); simple_mtx_init(&dev->vma_lock, mtx_plain); - util_vma_heap_init(&dev->main_heap, user_start, user_size); + agx_va_init_user(dev, user_start, user_size); util_vma_heap_init(&dev->usc_heap, dev->shader_base, shader_size); dev->vm_id = vm_create.vm_id; @@ -729,7 +729,7 @@ agx_close_device(struct agx_device *dev) util_sparse_array_finish(&dev->bo_map); agxdecode_destroy_context(dev->agxdecode); - util_vma_heap_finish(&dev->main_heap); + agx_va_finish_user(dev); util_vma_heap_finish(&dev->usc_heap); glsl_type_singleton_decref(); diff --git a/src/asahi/lib/agx_device.h b/src/asahi/lib/agx_device.h index e80dd9b5a5e1..7dcafc017cc3 100644 --- a/src/asahi/lib/agx_device.h +++ b/src/asahi/lib/agx_device.h @@ -120,6 +120,9 @@ struct agx_device { uint64_t shader_base; struct util_vma_heap main_heap; struct util_vma_heap usc_heap; + struct util_vma_heap capture_heap; + uint64_t capture_start; + uint64_t capture_size; uint64_t guard_size; /* To emulate sparse-resident buffers, we map buffers in both the bottom half @@ -272,6 +275,29 @@ struct agx_va *agx_va_alloc(struct agx_device *dev, uint64_t size_B, uint64_t fixed_va); void agx_va_free(struct agx_device *dev, struct agx_va *va, bool unbind); +/* The capture/replay range is a fixed slice at the bottom of the read-write + * user VA window, so it sits at the same place for every device on one machine + * and kernel, and a capture made in one process replays in another. It is + * finite: a capture/replay buffer, with its guard, must fit in 4 GiB. A larger + * one fails with VK_ERROR_OUT_OF_DEVICE_MEMORY, as any allocation that does not + * fit does. Ordinary buffers are not limited by it. + */ +#define AGX_CAPTURE_VA_SIZE_B (4ull << 30) + +/* Set up the read-write user heaps: the ordinary heap and the capture range + * carved out of the bottom of the window. Call with vma_lock initialised. + */ +void agx_va_init_user(struct agx_device *dev, uint64_t user_start, + uint64_t user_size); +void agx_va_finish_user(struct agx_device *dev); + +/* Can a capture/replay buffer of size_B (plus its guard) live at addr? That + * holds when the whole range, guard included, is inside the capture range and + * addr has the alignment. It does not say whether the range is free. + */ +bool agx_va_capture_valid(const struct agx_device *dev, uint64_t addr, + uint64_t size_B, uint64_t align_B); + static inline struct drm_asahi_cmd_header agx_cmd_header(bool compute, uint16_t barrier_vdm, uint16_t barrier_cdm) { diff --git a/src/asahi/lib/agx_va.c b/src/asahi/lib/agx_va.c index a2b56fc81465..b4ed8caa6964 100644 --- a/src/asahi/lib/agx_va.c +++ b/src/asahi/lib/agx_va.c @@ -9,7 +9,55 @@ static struct util_vma_heap * agx_vma_heap(struct agx_device *dev, enum agx_va_flags flags) { - return (flags & AGX_VA_USC) ? &dev->usc_heap : &dev->main_heap; + assert(!((flags & AGX_VA_USC) && (flags & AGX_VA_CAPTURE))); + + if (flags & AGX_VA_USC) + return &dev->usc_heap; + else if (flags & AGX_VA_CAPTURE) + return &dev->capture_heap; + else + return &dev->main_heap; +} + +void +agx_va_init_user(struct agx_device *dev, uint64_t user_start, + uint64_t user_size) +{ + /* A window smaller than four times the range keeps a quarter for capture + * and replay, so the ordinary heap is never empty. + */ + dev->capture_size = MIN2(AGX_CAPTURE_VA_SIZE_B, user_size / 4); + dev->capture_start = user_start; + + util_vma_heap_init(&dev->capture_heap, dev->capture_start, + dev->capture_size); + util_vma_heap_init(&dev->main_heap, user_start + dev->capture_size, + user_size - dev->capture_size); +} + +void +agx_va_finish_user(struct agx_device *dev) +{ + util_vma_heap_finish(&dev->main_heap); + util_vma_heap_finish(&dev->capture_heap); +} + +bool +agx_va_capture_valid(const struct agx_device *dev, uint64_t addr, + uint64_t size_B, uint64_t align_B) +{ + const uint64_t end = dev->capture_start + dev->capture_size; + + /* Written to never wrap: size_B comes from the application. */ + if (!align_B || size_B > end - dev->capture_start - dev->guard_size) + return false; + if (addr < dev->capture_start || addr >= end) + return false; + if (addr % align_B != 0) + return false; + + return addr - dev->capture_start <= + dev->capture_size - (size_B + dev->guard_size); } struct agx_va * @@ -19,6 +67,15 @@ agx_va_alloc(struct agx_device *dev, uint64_t size_B, uint64_t align_B, assert((fixed_va != 0) == !!(flags & AGX_VA_FIXED)); assert((fixed_va % align_B) == 0); + /* A fixed capture/replay request outside the capture range is not a request + * the heap can answer: util_vma_heap_alloc_addr asserts on a range that + * wraps. The caller checks agx_va_capture_valid to tell it from a range + * that is in use. + */ + if ((flags & AGX_VA_FIXED) && (flags & AGX_VA_CAPTURE) && + !agx_va_capture_valid(dev, fixed_va, size_B, align_B)) + return NULL; + /* All allocations need a guard at the end to prevent overreads. * * TODO: Even with soft fault? diff --git a/src/asahi/lib/meson.build b/src/asahi/lib/meson.build index e50a70eff518..e94ad8d1c84e 100644 --- a/src/asahi/lib/meson.build +++ b/src/asahi/lib/meson.build @@ -85,6 +85,20 @@ if with_tests suite : ['asahi'], protocol : 'gtest', ) + + test( + 'libasahi_va_tests', + executable( + 'libasahi_va_tests', + files('tests/test-agx-va.c'), + include_directories : [inc_asahi, inc_virtio_gpu, inc_virtio_vdrm], + c_args : [no_override_init_args, '-Wno-c2x-extensions'], + gnu_symbol_visibility : 'hidden', + dependencies: [dep_libdrm, idep_agx_pack, idep_mesautil, idep_libagx, idep_libpoly, idep_nir], + link_with : [libasahi_lib, libasahi_layout], + ), + suite : ['asahi'], + ) endif if dep_iokit.found() diff --git a/src/asahi/lib/tests/test-agx-va.c b/src/asahi/lib/tests/test-agx-va.c new file mode 100644 index 000000000000..0852e726facd --- /dev/null +++ b/src/asahi/lib/tests/test-agx-va.c @@ -0,0 +1,317 @@ +/* + * Copyright 2026 Joshua Warren + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include "agx_bo.h" +#include "agx_device.h" + +/* A window like the one agx_open_device builds: 64 GiB of user VA. */ +#define USER_START 0x2000000000ull +#define USER_SIZE 0x1000000000ull +#define PAGE 16384ull + +static unsigned failures; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, \ + #cond); \ + failures++; \ + } \ + } while (0) + +static void +open_va(struct agx_device *dev) +{ + memset(dev, 0, sizeof(*dev)); + simple_mtx_init(&dev->vma_lock, mtx_plain); + dev->guard_size = PAGE; + agx_va_init_user(dev, USER_START, USER_SIZE); +} + +static void +close_va(struct agx_device *dev) +{ + agx_va_finish_user(dev); + simple_mtx_destroy(&dev->vma_lock); +} + +static bool +overlaps(uint64_t a, uint64_t a_size, uint64_t b, uint64_t b_size) +{ + return a < b + b_size && b < a + a_size; +} + +static bool +in_capture(const struct agx_device *dev, const struct agx_va *va) +{ + return overlaps(va->addr, va->size_B, dev->capture_start, dev->capture_size); +} + +#define MAX_HELD 25000 +static struct agx_va *held[MAX_HELD]; +static unsigned nheld; + +static void +free_held(struct agx_device *dev) +{ + for (unsigned i = 0; i < nheld; ++i) + agx_va_free(dev, held[i], false); + nheld = 0; +} + +static void +test_range_shape(struct agx_device *dev) +{ + CHECK(dev->capture_size == AGX_CAPTURE_VA_SIZE_B); + CHECK(dev->capture_size == (4ull << 30)); + CHECK(dev->capture_start >= USER_START); + CHECK(dev->capture_start + dev->capture_size <= USER_START + USER_SIZE); +} + +static void +test_ordinary_stays_out(struct agx_device *dev) +{ + srand(1); + for (int i = 0; i < 20000; ++i) { + uint64_t size = PAGE << (rand() % 8); + struct agx_va *va = agx_va_alloc(dev, size, PAGE, 0, 0); + CHECK(va != NULL); + if (!va) + break; + CHECK(!in_capture(dev, va)); + held[nheld++] = va; + } + free_held(dev); +} + +static void +test_ordinary_stays_out_when_full(struct agx_device *dev) +{ + for (;;) { + struct agx_va *va = agx_va_alloc(dev, 1ull << 30, PAGE, 0, 0); + if (!va) + break; + CHECK(!in_capture(dev, va)); + held[nheld++] = va; + } + /* 64 GiB window minus the 4 GiB range, in (1 GiB + guard) steps. */ + CHECK(nheld > 50 && nheld < 61); + + /* The capture range is untouched and still usable. */ + struct agx_va *cap = agx_va_alloc(dev, 1ull << 30, PAGE, AGX_VA_CAPTURE, 0); + CHECK(cap != NULL); + agx_va_free(dev, cap, false); + free_held(dev); +} + +static void +test_capture_inside_with_guard(struct agx_device *dev) +{ + const uint64_t end = dev->capture_start + dev->capture_size; + + for (int i = 0; i < 100; ++i) { + uint64_t size = PAGE << (i % 7); + struct agx_va *va = agx_va_alloc(dev, size, PAGE, AGX_VA_CAPTURE, 0); + CHECK(va != NULL); + if (!va) + break; + CHECK(va->addr >= dev->capture_start); + CHECK(va->addr + va->size_B <= end); + CHECK(va->addr % PAGE == 0); + CHECK(va->size_B == size + dev->guard_size); + CHECK(va->flags & AGX_VA_CAPTURE); + for (unsigned j = 0; j < nheld; ++j) + CHECK(!overlaps(va->addr, va->size_B, held[j]->addr, held[j]->size_B)); + held[nheld++] = va; + } + free_held(dev); +} + +/* A capture in one device, a replay in a fresh one with ordinary allocations in + * between, as the conformance test does. + */ +static void +test_replay_in_fresh_device(struct agx_device *dev) +{ + uint64_t sizes[100], addrs[100]; + + srand(7); + for (int i = 0; i < 100; ++i) + sizes[i] = 4096ull << (rand() % 11); + + for (int i = 0; i < 100; ++i) { + uint64_t padded = (sizes[i] + PAGE - 1) & ~(PAGE - 1); + struct agx_va *va = agx_va_alloc(dev, padded, PAGE, AGX_VA_CAPTURE, 0); + CHECK(va != NULL); + if (!va) + return; + addrs[i] = va->addr; + held[nheld++] = va; + } + free_held(dev); + close_va(dev); + + open_va(dev); + for (int i = 99; i >= 0; --i) { + uint64_t padded = (sizes[i] + PAGE - 1) & ~(PAGE - 1); + CHECK(agx_va_capture_valid(dev, addrs[i], padded, PAGE)); + struct agx_va *va = agx_va_alloc( + dev, padded, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, addrs[i]); + CHECK(va != NULL); + if (!va) + continue; + CHECK(va->addr == addrs[i]); + held[nheld++] = va; + + struct agx_va *mem = agx_va_alloc(dev, padded, PAGE, 0, 0); + CHECK(mem != NULL); + if (mem) + held[nheld++] = mem; + } + free_held(dev); +} + +static void +test_invalid_requests(struct agx_device *dev) +{ + const uint64_t sz = PAGE * 4; + const uint64_t start = dev->capture_start; + const uint64_t end = start + dev->capture_size; + + CHECK(!agx_va_capture_valid(dev, end + PAGE, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, start - PAGE, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, end - PAGE, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, UINT64_MAX - PAGE + 1, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, 0, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, start + 4096, sz, PAGE)); + CHECK(!agx_va_capture_valid(dev, start, sz, 0)); + /* A size that would wrap, and one that leaves no room for the guard. */ + CHECK(!agx_va_capture_valid(dev, start, UINT64_MAX - 100, PAGE)); + CHECK(!agx_va_capture_valid(dev, start, dev->capture_size, PAGE)); + CHECK(!agx_va_capture_valid(dev, end - sz, sz, PAGE)); + CHECK(agx_va_capture_valid(dev, end - sz - dev->guard_size, sz, PAGE)); + CHECK(agx_va_capture_valid(dev, start, sz, PAGE)); + + CHECK(agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, + end + PAGE) == NULL); + CHECK(agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, + UINT64_MAX - PAGE + 1) == NULL); +} + +static void +test_occupied_request(struct agx_device *dev) +{ + const uint64_t sz = PAGE * 4; + struct agx_va *a = agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE, 0); + CHECK(a != NULL); + if (!a) + return; + const uint64_t addr = a->addr; + + CHECK(agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, addr) == + NULL); + /* The guard page after the allocation is occupied too. */ + CHECK(agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, + addr + sz) == NULL); + + agx_va_free(dev, a, false); + struct agx_va *again = + agx_va_alloc(dev, sz, PAGE, AGX_VA_CAPTURE | AGX_VA_FIXED, addr); + CHECK(again != NULL); + if (again) { + CHECK(again->addr == addr); + agx_va_free(dev, again, false); + } +} + +static unsigned +fill_capture(struct agx_device *dev) +{ + unsigned n = 0; + for (;;) { + struct agx_va *va = agx_va_alloc(dev, 1ull << 28, PAGE, AGX_VA_CAPTURE, 0); + if (!va) + return n; + held[nheld++] = va; + n++; + } +} + +static void +test_exhaustion_and_reuse(struct agx_device *dev) +{ + /* 4 GiB in (256 MiB + guard) steps. */ + CHECK(fill_capture(dev) == 15); + + struct agx_va *ord = agx_va_alloc(dev, 1ull << 30, PAGE, 0, 0); + CHECK(ord != NULL); + if (ord) { + CHECK(!in_capture(dev, ord)); + agx_va_free(dev, ord, false); + } + + /* Freeing returns every range to the capture heap. */ + free_held(dev); + CHECK(fill_capture(dev) == 15); + free_held(dev); +} + +static void +test_freed_capture_range_stays_reserved(struct agx_device *dev) +{ + struct agx_va *cap = agx_va_alloc(dev, PAGE * 8, PAGE, AGX_VA_CAPTURE, 0); + CHECK(cap != NULL); + if (!cap) + return; + const uint64_t addr = cap->addr; + agx_va_free(dev, cap, false); + + for (int i = 0; i < 5000; ++i) { + struct agx_va *va = agx_va_alloc(dev, PAGE, PAGE, 0, 0); + CHECK(va != NULL); + if (!va) + break; + CHECK(va->addr != addr); + CHECK(!in_capture(dev, va)); + held[nheld++] = va; + } + free_held(dev); +} + +int +main(void) +{ + static const struct { + const char *name; + void (*fn)(struct agx_device *); + } tests[] = { + {"range shape", test_range_shape}, + {"ordinary stays out", test_ordinary_stays_out}, + {"ordinary stays out when full", test_ordinary_stays_out_when_full}, + {"capture inside with guard", test_capture_inside_with_guard}, + {"replay in fresh device", test_replay_in_fresh_device}, + {"invalid requests", test_invalid_requests}, + {"occupied request", test_occupied_request}, + {"exhaustion and reuse", test_exhaustion_and_reuse}, + {"freed capture range stays reserved", + test_freed_capture_range_stays_reserved}, + }; + static struct agx_device dev; + + for (unsigned i = 0; i < sizeof(tests) / sizeof(tests[0]); ++i) { + unsigned before = failures; + open_va(&dev); + tests[i].fn(&dev); + close_va(&dev); + printf("%s: %s\n", failures == before ? "PASS" : "FAIL", tests[i].name); + } + + printf("%u failure(s)\n", failures); + return failures != 0; +} diff --git a/src/asahi/vulkan/hk_buffer.c b/src/asahi/vulkan/hk_buffer.c index 8ec78e9a7e82..4acf8b156fe2 100644 --- a/src/asahi/vulkan/hk_buffer.c +++ b/src/asahi/vulkan/hk_buffer.c @@ -6,6 +6,8 @@ */ #include "hk_buffer.h" +#include + #include "agx_bo.h" #include "agx_device.h" #include "hk_device.h" @@ -128,12 +130,28 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT; - enum agx_va_flags flags = 0; + /* Capture/replay buffers draw from their own VA range, which no other + * allocation uses, so a captured address is free again when it is + * replayed. + */ + enum agx_va_flags flags = bda_capture_replay ? AGX_VA_CAPTURE : 0; uint64_t bda_fixed_addr = 0; if (bda_capture_replay) { bda_fixed_addr = hk_get_bda_replay_addr(pCreateInfo); - if (bda_fixed_addr != 0) + if (bda_fixed_addr != 0) { + /* An address that cannot be in the range is not available, which + * is not the same as running out of address space. + */ + if (!agx_va_capture_valid(&dev->dev, bda_fixed_addr, vma_size_B, + alignment)) { + vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk); + return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS, + "Capture address 0x%" PRIx64 + " is outside the capture range", + bda_fixed_addr); + } flags |= AGX_VA_FIXED; + } } buffer->va = @@ -141,6 +159,16 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, if (!buffer->va) { vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk); + + /* A fixed request has one place to go. If it fails the address is in + * use. A request without an address fails when the range is full. + */ + if (bda_fixed_addr != 0) { + return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS, + "Capture address 0x%" PRIx64 " is in use", + bda_fixed_addr); + } + return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "Sparse VMA allocation failed"); }