drivers: hv: dxgkrnl: Operations using sync objects

Implement ioctls to submit operations with compute device
sync objects:
  - the LX_DXSIGNALSYNCHRONIZATIONOBJECT ioctl.
    The ioctl is used to submit a signal to a sync object.
  - the LX_DXWAITFORSYNCHRONIZATIONOBJECT ioctl.
    The ioctl is used to submit a wait for a sync object
  - the LX_DXSIGNALSYNCHRONIZATIONOBJECTFROMCPU ioctl
    The ioctl is used to signal to a monitored fence sync object
    from a CPU thread.
  - the LX_DXSIGNALSYNCHRONIZATIONOBJECTFROMGPU ioctl.
    The ioctl is used to submit a signal to a monitored fence
    sync object..
  - the LX_DXSIGNALSYNCHRONIZATIONOBJECTFROMGPU2 ioctl.
    The ioctl is used to submit a signal to a monitored fence
    sync object.
  - the LX_DXWAITFORSYNCHRONIZATIONOBJECTFROMGPU ioctl.
    The ioctl is used to submit a wait for a monitored fence
    sync object.

Compute device synchronization objects are used to synchronize
execution of DMA buffers between different execution contexts.
Operations with sync objects include "signal" and "wait". A wait
for a sync object is satisfied when the sync object is signaled.

A signal operation could be submitted to a compute device context or
the sync object could be signaled by a CPU thread.

To improve performance, submitting operations to the host is done
asynchronously when the host supports it.

Signed-off-by: Iouri Tarassov <iourit@linux.microsoft.com>
[kms: forward port to 6.6 from 6.1. No code changes made.]
Signed-off-by: Kelsey Steele <kelseysteele@microsoft.com>

Author: Iouri Tarassov

drivers/hv/dxgkrnl/dxgadapter.c

@@ -249,7 +249,7 @@ void dxgdevice_stop(struct dxgdevice *device)
 	struct dxgallocation *alloc;
 	struct dxgsyncobject *syncobj;
 
-	DXG_TRACE("Destroying device: %p", device);
+	DXG_TRACE("Stopping device: %p", device);
 	dxgdevice_acquire_alloc_list_lock(device);
 	list_for_each_entry(alloc, &device->alloc_list_head, alloc_list_entry) {
 		dxgallocation_stop(alloc);
@@ -743,15 +743,13 @@ void dxgallocation_destroy(struct dxgallocation *alloc)
 	}
 #ifdef _MAIN_KERNEL_
 	if (alloc->gpadl.gpadl_handle) {
-		DXG_TRACE("Teardown gpadl %d",
-			alloc->gpadl.gpadl_handle);
+		DXG_TRACE("Teardown gpadl %d", alloc->gpadl.gpadl_handle);
 		vmbus_teardown_gpadl(dxgglobal_get_vmbus(), &alloc->gpadl);
 		alloc->gpadl.gpadl_handle = 0;
 	}
 else
 	if (alloc->gpadl) {
-		DXG_TRACE("Teardown gpadl %d",
-			alloc->gpadl);
+		DXG_TRACE("Teardown gpadl %d", alloc->gpadl);
 		vmbus_teardown_gpadl(dxgglobal_get_vmbus(), alloc->gpadl);
 		alloc->gpadl = 0;
 	}
@@ -901,6 +899,13 @@ struct dxgsyncobject *dxgsyncobject_create(struct dxgprocess *process,
 	case _D3DDDI_PERIODIC_MONITORED_FENCE:
 		syncobj->monitored_fence = 1;
 		break;
+	case _D3DDDI_CPU_NOTIFICATION:
+		syncobj->cpu_event = 1;
+		syncobj->host_event = kzalloc(sizeof(*syncobj->host_event),
+					GFP_KERNEL);
+		if (syncobj->host_event == NULL)
+			goto cleanup;
+		break;
 	default:
 		break;
 	}
@@ -928,6 +933,8 @@ struct dxgsyncobject *dxgsyncobject_create(struct dxgprocess *process,
 	DXG_TRACE("Syncobj created: %p", syncobj);
 	return syncobj;
 cleanup:
+	if (syncobj->host_event)
+		kfree(syncobj->host_event);
 	if (syncobj)
 		kfree(syncobj);
 	return NULL;
@@ -937,6 +944,7 @@ void dxgsyncobject_destroy(struct dxgprocess *process,
 			   struct dxgsyncobject *syncobj)
 {
 	int destroyed;
+	struct dxghosteventcpu *host_event;
 
 	DXG_TRACE("Destroying syncobj: %p", syncobj);
 
@@ -955,6 +963,16 @@ void dxgsyncobject_destroy(struct dxgprocess *process,
 		}
 		hmgrtable_unlock(&process->handle_table, DXGLOCK_EXCL);
 
+		if (syncobj->cpu_event) {
+			host_event = syncobj->host_event;
+			if (host_event->cpu_event) {
+				eventfd_ctx_put(host_event->cpu_event);
+				if (host_event->hdr.event_id)
+					dxgglobal_remove_host_event(
+						&host_event->hdr);
+				host_event->cpu_event = NULL;
+			}
+		}
 		if (syncobj->monitored_fence)
 			dxgdevice_remove_syncobj(syncobj);
 		else
@@ -971,16 +989,14 @@ void dxgsyncobject_destroy(struct dxgprocess *process,
 void dxgsyncobject_stop(struct dxgsyncobject *syncobj)
 {
 	int stopped = test_and_set_bit(1, &syncobj->flags);
+	int ret;
 
 	if (!stopped) {
 		DXG_TRACE("Stopping syncobj");
 		if (syncobj->monitored_fence) {
 			if (syncobj->mapped_address) {
-				int ret =
-				    dxg_unmap_iospace(syncobj->mapped_address,
-						      PAGE_SIZE);
-
-				(void)ret;
+				ret = dxg_unmap_iospace(syncobj->mapped_address,
+							PAGE_SIZE);
 				DXG_TRACE("unmap fence %d %p",
 					ret, syncobj->mapped_address);
 				syncobj->mapped_address = NULL;
@@ -994,5 +1010,7 @@ void dxgsyncobject_release(struct kref *refcount)
 	struct dxgsyncobject *syncobj;
 
 	syncobj = container_of(refcount, struct dxgsyncobject, syncobj_kref);
+	if (syncobj->host_event)
+		kfree(syncobj->host_event);
 	kfree(syncobj);
 }

drivers/hv/dxgkrnl/dxgkrnl.h

@@ -101,6 +101,29 @@ int dxgvmbuschannel_init(struct dxgvmbuschannel *ch, struct hv_device *hdev);
 void dxgvmbuschannel_destroy(struct dxgvmbuschannel *ch);
 void dxgvmbuschannel_receive(void *ctx);
 
+/*
+ * The structure describes an event, which will be signaled by
+ * a message from host.
+ */
+enum dxghosteventtype {
+	dxghostevent_cpu_event = 1,
+};
+
+struct dxghostevent {
+	struct list_head	host_event_list_entry;
+	u64			event_id;
+	enum dxghosteventtype	event_type;
+};
+
+struct dxghosteventcpu {
+	struct dxghostevent	hdr;
+	struct dxgprocess	*process;
+	struct eventfd_ctx	*cpu_event;
+	struct completion	*completion_event;
+	bool			destroy_after_signal;
+	bool			remove_from_list;
+};
+
 /*
  * This is GPU synchronization object, which is used to synchronize execution
  * between GPU contextx/hardware queues or for tracking GPU execution progress.
@@ -130,6 +153,8 @@ struct dxgsyncobject {
 	 */
 	struct dxgdevice		*device;
 	struct dxgprocess		*process;
+	/* Used by D3DDDI_CPU_NOTIFICATION objects */
+	struct dxghosteventcpu		*host_event;
 	/* CPU virtual address of the fence value for "device" syncobjects */
 	void				*mapped_address;
 	/* Handle in the process handle table */
@@ -144,6 +169,7 @@ struct dxgsyncobject {
 			u32		stopped:1;
 			/* device syncobject */
 			u32		monitored_fence:1;
+			u32		cpu_event:1;
 			u32		shared:1;
 			u32		reserved:27;
 		};
@@ -206,6 +232,11 @@ struct dxgglobal {
 	/* protects the dxgprocess_adapter lists */
 	struct mutex		process_adapter_mutex;
 
+	/*  list of events, waiting to be signaled by the host */
+	struct list_head	host_event_list_head;
+	spinlock_t		host_event_list_mutex;
+	atomic64_t		host_event_id;
+
 	bool			global_channel_initialized;
 	bool			async_msg_enabled;
 	bool			misc_registered;
@@ -228,6 +259,11 @@ struct vmbus_channel *dxgglobal_get_vmbus(void);
 struct dxgvmbuschannel *dxgglobal_get_dxgvmbuschannel(void);
 void dxgglobal_acquire_process_adapter_lock(void);
 void dxgglobal_release_process_adapter_lock(void);
+void dxgglobal_add_host_event(struct dxghostevent *hostevent);
+void dxgglobal_remove_host_event(struct dxghostevent *hostevent);
+u64 dxgglobal_new_host_event_id(void);
+void dxgglobal_signal_host_event(u64 event_id);
+struct dxghostevent *dxgglobal_get_host_event(u64 event_id);
 int dxgglobal_acquire_channel_lock(void);
 void dxgglobal_release_channel_lock(void);
 
@@ -594,6 +630,31 @@ int dxgvmb_send_create_sync_object(struct dxgprocess *pr,
 				   *args, struct dxgsyncobject *so);
 int dxgvmb_send_destroy_sync_object(struct dxgprocess *pr,
 				    struct d3dkmthandle h);
+int dxgvmb_send_signal_sync_object(struct dxgprocess *process,
+				   struct dxgadapter *adapter,
+				   struct d3dddicb_signalflags flags,
+				   u64 legacy_fence_value,
+				   struct d3dkmthandle context,
+				   u32 object_count,
+				   struct d3dkmthandle *object,
+				   u32 context_count,
+				   struct d3dkmthandle *contexts,
+				   u32 fence_count, u64 *fences,
+				   struct eventfd_ctx *cpu_event,
+				   struct d3dkmthandle device);
+int dxgvmb_send_wait_sync_object_gpu(struct dxgprocess *process,
+				     struct dxgadapter *adapter,
+				     struct d3dkmthandle context,
+				     u32 object_count,
+				     struct d3dkmthandle *objects,
+				     u64 *fences,
+				     bool legacy_fence);
+int dxgvmb_send_wait_sync_object_cpu(struct dxgprocess *process,
+				     struct dxgadapter *adapter,
+				     struct
+				     d3dkmt_waitforsynchronizationobjectfromcpu
+				     *args,
+				     u64 cpu_event);
 int dxgvmb_send_query_adapter_info(struct dxgprocess *process,
 				   struct dxgadapter *adapter,
 				   struct d3dkmt_queryadapterinfo *args);
@@ -609,6 +670,7 @@ int dxgvmb_send_async_msg(struct dxgvmbuschannel *channel,
 			  void *command,
 			  u32 cmd_size);
 
+void signal_host_cpu_event(struct dxghostevent *eventhdr);
 int ntstatus2int(struct ntstatus status);
 
 #ifdef DEBUG

drivers/hv/dxgkrnl/dxgmodule.c

@@ -123,6 +123,102 @@ static struct dxgadapter *find_adapter(struct winluid *luid)
 	return adapter;
 }
 
+void dxgglobal_add_host_event(struct dxghostevent *event)
+{
+	struct dxgglobal *dxgglobal = dxggbl();
+
+	spin_lock_irq(&dxgglobal->host_event_list_mutex);
+	list_add_tail(&event->host_event_list_entry,
+		      &dxgglobal->host_event_list_head);
+	spin_unlock_irq(&dxgglobal->host_event_list_mutex);
+}
+
+void dxgglobal_remove_host_event(struct dxghostevent *event)
+{
+	struct dxgglobal *dxgglobal = dxggbl();
+
+	spin_lock_irq(&dxgglobal->host_event_list_mutex);
+	if (event->host_event_list_entry.next != NULL) {
+		list_del(&event->host_event_list_entry);
+		event->host_event_list_entry.next = NULL;
+	}
+	spin_unlock_irq(&dxgglobal->host_event_list_mutex);
+}
+
+void signal_host_cpu_event(struct dxghostevent *eventhdr)
+{
+	struct  dxghosteventcpu *event = (struct  dxghosteventcpu *)eventhdr;
+
+	if (event->remove_from_list ||
+		event->destroy_after_signal) {
+		list_del(&eventhdr->host_event_list_entry);
+		eventhdr->host_event_list_entry.next = NULL;
+	}
+	if (event->cpu_event) {
+		DXG_TRACE("signal cpu event");
+		eventfd_signal(event->cpu_event, 1);
+		if (event->destroy_after_signal)
+			eventfd_ctx_put(event->cpu_event);
+	} else {
+		DXG_TRACE("signal completion");
+		complete(event->completion_event);
+	}
+	if (event->destroy_after_signal) {
+		DXG_TRACE("destroying event %p", event);
+		kfree(event);
+	}
+}
+
+void dxgglobal_signal_host_event(u64 event_id)
+{
+	struct dxghostevent *event;
+	unsigned long flags;
+	struct dxgglobal *dxgglobal = dxggbl();
+
+	DXG_TRACE("Signaling host event %lld", event_id);
+
+	spin_lock_irqsave(&dxgglobal->host_event_list_mutex, flags);
+	list_for_each_entry(event, &dxgglobal->host_event_list_head,
+			    host_event_list_entry) {
+		if (event->event_id == event_id) {
+			DXG_TRACE("found event to signal");
+			if (event->event_type == dxghostevent_cpu_event)
+				signal_host_cpu_event(event);
+			else
+				DXG_ERR("Unknown host event type");
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&dxgglobal->host_event_list_mutex, flags);
+}
+
+struct dxghostevent *dxgglobal_get_host_event(u64 event_id)
+{
+	struct dxghostevent *entry;
+	struct dxghostevent *event = NULL;
+	struct dxgglobal *dxgglobal = dxggbl();
+
+	spin_lock_irq(&dxgglobal->host_event_list_mutex);
+	list_for_each_entry(entry, &dxgglobal->host_event_list_head,
+			    host_event_list_entry) {
+		if (entry->event_id == event_id) {
+			list_del(&entry->host_event_list_entry);
+			entry->host_event_list_entry.next = NULL;
+			event = entry;
+			break;
+		}
+	}
+	spin_unlock_irq(&dxgglobal->host_event_list_mutex);
+	return event;
+}
+
+u64 dxgglobal_new_host_event_id(void)
+{
+	struct dxgglobal *dxgglobal = dxggbl();
+
+	return atomic64_inc_return(&dxgglobal->host_event_id);
+}
+
 void dxgglobal_acquire_process_adapter_lock(void)
 {
 	struct dxgglobal *dxgglobal = dxggbl();
@@ -720,12 +816,16 @@ static struct dxgglobal *dxgglobal_create(void)
 	INIT_LIST_HEAD(&dxgglobal->vgpu_ch_list_head);
 	INIT_LIST_HEAD(&dxgglobal->adapter_list_head);
 	init_rwsem(&dxgglobal->adapter_list_lock);
-
 	init_rwsem(&dxgglobal->channel_lock);
 
+	INIT_LIST_HEAD(&dxgglobal->host_event_list_head);
+	spin_lock_init(&dxgglobal->host_event_list_mutex);
+	atomic64_set(&dxgglobal->host_event_id, 1);
+
 #ifdef DEBUG
 	dxgk_validate_ioctls();
 #endif
+
 	return dxgglobal;
 }