honeykrisp: give capture and replay buffers their own address range

src/asahi/lib/agx_bo.h

@@ -45,6 +45,13 @@ enum agx_va_flags {
 
    /* VA must be fixed, otherwise allocated by the driver. */
    AGX_VA_FIXED = (1 << 1),
+
+   /* VA must be inside the capture/replay range. Only buffers created with
+    * VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT use it, and no other
+    * allocation draws from it, so a captured address stays free for the
+    * replay. Not allowed together with AGX_VA_USC.
+    */
+   AGX_VA_CAPTURE = (1 << 2),
 };
 
 struct agx_va {

src/asahi/lib/agx_device.c

@@ -646,7 +646,7 @@ agx_open_device(void *memctx, struct agx_device *dev)
    assert(((user_start + (user_size - 1)) & dev->sparse_ro_offset) == 0);
 
    simple_mtx_init(&dev->vma_lock, mtx_plain);
-   util_vma_heap_init(&dev->main_heap, user_start, user_size);
+   agx_va_init_user(dev, user_start, user_size);
    util_vma_heap_init(&dev->usc_heap, dev->shader_base, shader_size);
 
    dev->vm_id = vm_create.vm_id;
@@ -729,7 +729,7 @@ agx_close_device(struct agx_device *dev)
    util_sparse_array_finish(&dev->bo_map);
    agxdecode_destroy_context(dev->agxdecode);
 
-   util_vma_heap_finish(&dev->main_heap);
+   agx_va_finish_user(dev);
    util_vma_heap_finish(&dev->usc_heap);
    glsl_type_singleton_decref();
 

src/asahi/lib/agx_device.h

@@ -120,6 +120,9 @@ struct agx_device {
    uint64_t shader_base;
    struct util_vma_heap main_heap;
    struct util_vma_heap usc_heap;
+   struct util_vma_heap capture_heap;
+   uint64_t capture_start;
+   uint64_t capture_size;
    uint64_t guard_size;
 
    /* To emulate sparse-resident buffers, we map buffers in both the bottom half
@@ -272,6 +275,29 @@ struct agx_va *agx_va_alloc(struct agx_device *dev, uint64_t size_B,
                             uint64_t fixed_va);
 void agx_va_free(struct agx_device *dev, struct agx_va *va, bool unbind);
 
+/* The capture/replay range is a fixed slice at the bottom of the read-write
+ * user VA window, so it sits at the same place for every device on one machine
+ * and kernel, and a capture made in one process replays in another. It is
+ * finite: a capture/replay buffer, with its guard, must fit in 4 GiB. A larger
+ * one fails with VK_ERROR_OUT_OF_DEVICE_MEMORY, as any allocation that does not
+ * fit does. Ordinary buffers are not limited by it.
+ */
+#define AGX_CAPTURE_VA_SIZE_B (4ull << 30)
+
+/* Set up the read-write user heaps: the ordinary heap and the capture range
+ * carved out of the bottom of the window. Call with vma_lock initialised.
+ */
+void agx_va_init_user(struct agx_device *dev, uint64_t user_start,
+                      uint64_t user_size);
+void agx_va_finish_user(struct agx_device *dev);
+
+/* Can a capture/replay buffer of size_B (plus its guard) live at addr? That
+ * holds when the whole range, guard included, is inside the capture range and
+ * addr has the alignment. It does not say whether the range is free.
+ */
+bool agx_va_capture_valid(const struct agx_device *dev, uint64_t addr,
+                          uint64_t size_B, uint64_t align_B);
+
 static inline struct drm_asahi_cmd_header
 agx_cmd_header(bool compute, uint16_t barrier_vdm, uint16_t barrier_cdm)
 {

src/asahi/lib/agx_va.c

@@ -9,7 +9,55 @@
 static struct util_vma_heap *
 agx_vma_heap(struct agx_device *dev, enum agx_va_flags flags)
 {
-   return (flags & AGX_VA_USC) ? &dev->usc_heap : &dev->main_heap;
+   assert(!((flags & AGX_VA_USC) && (flags & AGX_VA_CAPTURE)));
+
+   if (flags & AGX_VA_USC)
+      return &dev->usc_heap;
+   else if (flags & AGX_VA_CAPTURE)
+      return &dev->capture_heap;
+   else
+      return &dev->main_heap;
+}
+
+void
+agx_va_init_user(struct agx_device *dev, uint64_t user_start,
+                 uint64_t user_size)
+{
+   /* A window smaller than four times the range keeps a quarter for capture
+    * and replay, so the ordinary heap is never empty.
+    */
+   dev->capture_size = MIN2(AGX_CAPTURE_VA_SIZE_B, user_size / 4);
+   dev->capture_start = user_start;
+
+   util_vma_heap_init(&dev->capture_heap, dev->capture_start,
+                      dev->capture_size);
+   util_vma_heap_init(&dev->main_heap, user_start + dev->capture_size,
+                      user_size - dev->capture_size);
+}
+
+void
+agx_va_finish_user(struct agx_device *dev)
+{
+   util_vma_heap_finish(&dev->main_heap);
+   util_vma_heap_finish(&dev->capture_heap);
+}
+
+bool
+agx_va_capture_valid(const struct agx_device *dev, uint64_t addr,
+                     uint64_t size_B, uint64_t align_B)
+{
+   const uint64_t end = dev->capture_start + dev->capture_size;
+
+   /* Written to never wrap: size_B comes from the application. */
+   if (size_B > end - dev->capture_start - dev->guard_size)
+      return false;
+   if (addr < dev->capture_start || addr >= end)
+      return false;
+   if (addr % align_B != 0)
+      return false;
+
+   return addr - dev->capture_start <=
+          dev->capture_size - (size_B + dev->guard_size);
 }
 
 struct agx_va *
@@ -19,6 +67,15 @@ agx_va_alloc(struct agx_device *dev, uint64_t size_B, uint64_t align_B,
    assert((fixed_va != 0) == !!(flags & AGX_VA_FIXED));
    assert((fixed_va % align_B) == 0);
 
+   /* A fixed capture/replay request outside the capture range is not a request
+    * the heap can answer: util_vma_heap_alloc_addr asserts on a range that
+    * wraps. The caller checks agx_va_capture_valid to tell it from a range
+    * that is in use.
+    */
+   if ((flags & AGX_VA_FIXED) && (flags & AGX_VA_CAPTURE) &&
+       !agx_va_capture_valid(dev, fixed_va, size_B, align_B))
+      return NULL;
+
    /* All allocations need a guard at the end to prevent overreads.
     *
     * TODO: Even with soft fault?

src/asahi/lib/meson.build

@@ -85,6 +85,20 @@ if with_tests
     suite : ['asahi'],
     protocol : 'gtest',
   )
+
+  test(
+    'libasahi_va_tests',
+    executable(
+      'libasahi_va_tests',
+      files('tests/test-agx-va.c'),
+      include_directories : [inc_asahi, inc_virtio_gpu, inc_virtio_vdrm],
+      c_args : [no_override_init_args, '-Wno-c2x-extensions'],
+      gnu_symbol_visibility : 'hidden',
+      dependencies: [dep_libdrm, idep_agx_pack, idep_mesautil, idep_libagx, idep_libpoly],
+      link_with : [libasahi_lib, libasahi_layout],
+    ),
+    suite : ['asahi'],
+  )
 endif
 
 if dep_iokit.found()