drivers: hv: dxgkrnl: Added implementation for D3DKMTInvalidateCache

D3DKMTInvalidateCache is called by user mode drivers when the device
doesn't support cache coherent access to compute device allocations.
It needs to be called after an allocation was accessed by CPU and now
needs to be accessed by the device. And vice versa.

Signed-off-by: Iouri Tarassov <iourit@linux.microsoft.com>
[kms: forward port to 6.6 from 6.1. No code changes made.]
Signed-off-by: Kelsey Steele <kelseysteele@microsoft.com>

Author: Iouri Tarassov

drivers/hv/dxgkrnl/dxgkrnl.h

@@ -989,6 +989,9 @@ int dxgvmb_send_async_msg(struct dxgvmbuschannel *channel,
 			  u32 cmd_size);
 int dxgvmb_send_share_object_with_host(struct dxgprocess *process,
 				struct d3dkmt_shareobjectwithhost *args);
+int dxgvmb_send_invalidate_cache(struct dxgprocess *process,
+				struct dxgadapter *adapter,
+				struct d3dkmt_invalidatecache *args);
 
 void signal_host_cpu_event(struct dxghostevent *eventhdr);
 int ntstatus2int(struct ntstatus status);

drivers/hv/dxgkrnl/dxgvmbus.c

@@ -2021,6 +2021,33 @@ int dxgvmb_send_flush_heap_transitions(struct dxgprocess *process,
 	return ret;
 }
 
+int dxgvmb_send_invalidate_cache(struct dxgprocess *process,
+				struct dxgadapter *adapter,
+				struct d3dkmt_invalidatecache *args)
+{
+	struct dxgkvmb_command_invalidatecache *command;
+	int ret;
+	struct dxgvmbusmsg msg = {.hdr = NULL};
+
+	ret = init_message(&msg, adapter, process, sizeof(*command));
+	if (ret)
+		goto cleanup;
+	command = (void *)msg.msg;
+	command_vgpu_to_host_init2(&command->hdr,
+				   DXGK_VMBCOMMAND_INVALIDATECACHE,
+				   process->host_handle);
+	command->device = args->device;
+	command->allocation = args->allocation;
+	command->offset = args->offset;
+	command->length = args->length;
+	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
+cleanup:
+	free_message(&msg, process);
+	if (ret)
+		DXG_TRACE("err: %d", ret);
+	return ret;
+}
+
 int dxgvmb_send_query_alloc_residency(struct dxgprocess *process,
 				      struct dxgadapter *adapter,
 				      struct d3dkmt_queryallocationresidency

drivers/hv/dxgkrnl/dxgvmbus.h

@@ -125,6 +125,7 @@ enum dxgkvmb_commandtype {
 	DXGK_VMBCOMMAND_QUERYRESOURCEINFO	= 64,
 	DXGK_VMBCOMMAND_LOGEVENT		= 65,
 	DXGK_VMBCOMMAND_SETEXISTINGSYSMEMPAGES	= 66,
+	DXGK_VMBCOMMAND_INVALIDATECACHE		= 67,
 	DXGK_VMBCOMMAND_INVALID
 };
 
@@ -428,6 +429,16 @@ struct dxgkvmb_command_flushheaptransitions {
 	struct dxgkvmb_command_vgpu_to_host hdr;
 };
 
+/* Returns  ntstatus */
+struct dxgkvmb_command_invalidatecache {
+	struct dxgkvmb_command_vgpu_to_host hdr;
+	struct d3dkmthandle device;
+	struct d3dkmthandle allocation;
+	u64 offset;
+	u64 length;
+	u64 reserved;
+};
+
 struct dxgkvmb_command_freegpuvirtualaddress {
 	struct dxgkvmb_command_vgpu_to_host hdr;
 	struct d3dkmt_freegpuvirtualaddress args;

drivers/hv/dxgkrnl/ioctl.c

@@ -4286,6 +4286,8 @@ dxgkio_query_clock_calibration(struct dxgprocess *process, void *__user inargs)
 		dxgadapter_release_lock_shared(adapter);
 	if (adapter)
 		kref_put(&adapter->adapter_kref, dxgadapter_release);
+
+	DXG_TRACE_IOCTL_END(ret);
 	return ret;
 }
 
@@ -4333,6 +4335,49 @@ dxgkio_flush_heap_transitions(struct dxgprocess *process, void *__user inargs)
 		dxgadapter_release_lock_shared(adapter);
 	if (adapter)
 		kref_put(&adapter->adapter_kref, dxgadapter_release);
+
+	DXG_TRACE_IOCTL_END(ret);
+	return ret;
+}
+
+static int
+dxgkio_invalidate_cache(struct dxgprocess *process, void *__user inargs)
+{
+	struct d3dkmt_invalidatecache args;
+	int ret;
+	struct dxgdevice *device = NULL;
+
+	ret = copy_from_user(&args, inargs, sizeof(args));
+	if (ret) {
+		DXG_ERR("failed to copy input args");
+		ret = -EFAULT;
+		goto cleanup;
+	}
+
+	device = dxgprocess_device_by_handle(process, args.device);
+	if (device == NULL) {
+		ret = -EINVAL;
+		goto cleanup;
+	}
+
+	ret = dxgdevice_acquire_lock_shared(device);
+	if (ret < 0) {
+		kref_put(&device->device_kref, dxgdevice_release);
+		device = NULL;
+		goto cleanup;
+	}
+
+	ret = dxgvmb_send_invalidate_cache(process, device->adapter,
+		&args);
+
+cleanup:
+
+	if (device) {
+		dxgdevice_release_lock_shared(device);
+		kref_put(&device->device_kref, dxgdevice_release);
+	}
+
+	DXG_TRACE_IOCTL_END(ret);
 	return ret;
 }
 
@@ -5198,7 +5243,7 @@ static struct ioctl_desc ioctls[] = {
 /* 0x22 */	{dxgkio_get_context_scheduling_priority,
 		 LX_DXGETCONTEXTSCHEDULINGPRIORITY},
 /* 0x23 */	{},
-/* 0x24 */	{},
+/* 0x24 */	{dxgkio_invalidate_cache, LX_DXINVALIDATECACHE},
 /* 0x25 */	{dxgkio_lock2, LX_DXLOCK2},
 /* 0x26 */	{dxgkio_mark_device_as_error, LX_DXMARKDEVICEASERROR},
 /* 0x27 */	{dxgkio_offer_allocations, LX_DXOFFERALLOCATIONS},
@@ -5243,7 +5288,7 @@ static struct ioctl_desc ioctls[] = {
 /* 0x44 */	{dxgkio_share_object_with_host, LX_DXSHAREOBJECTWITHHOST},
 /* 0x45 */	{dxgkio_create_sync_file, LX_DXCREATESYNCFILE},
 /* 0x46 */	{dxgkio_wait_sync_file, LX_DXWAITSYNCFILE},
-/* 0x46 */	{dxgkio_open_syncobj_from_syncfile,
+/* 0x47 */	{dxgkio_open_syncobj_from_syncfile,
 		 LX_DXOPENSYNCOBJECTFROMSYNCFILE},
 };
 

include/uapi/misc/d3dkmthk.h

@@ -1580,6 +1580,13 @@ struct d3dkmt_opensyncobjectfromsyncfile {
 	__u64			fence_value_gpu_va;	/* out */
 };
 
+struct d3dkmt_invalidatecache {
+	struct d3dkmthandle	device;
+	struct d3dkmthandle	allocation;
+	__u64			offset;
+	__u64			length;
+};
+
 /*
  * Dxgkrnl Graphics Port Driver ioctl definitions
  *
@@ -1647,6 +1654,8 @@ struct d3dkmt_opensyncobjectfromsyncfile {
 	_IOWR(0x47, 0x21, struct d3dkmt_getcontextinprocessschedulingpriority)
 #define LX_DXGETCONTEXTSCHEDULINGPRIORITY \
 	_IOWR(0x47, 0x22, struct d3dkmt_getcontextschedulingpriority)
+#define LX_DXINVALIDATECACHE \
+	_IOWR(0x47, 0x24, struct d3dkmt_invalidatecache)
 #define LX_DXLOCK2			\
 	_IOWR(0x47, 0x25, struct d3dkmt_lock2)
 #define LX_DXMARKDEVICEASERROR		\