android10/kernel/drivers/misc/google-easel-comm-dma.c

/*
 * Android/Easel coprocessor communication DMA routines.
 *
 * Copyright 2016 Google Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

/* #define DEBUG */

#include <uapi/linux/google-easel-comm.h>
#include "google-easel-comm-shared.h"
#include "google-easel-comm-private.h"

#include <linux/uaccess.h>
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>

/*
 * Server receives DMA scatter-gather list from client.  Store this and
 * wake up the local process handling the DMA request.
 */
void easelcomm_handle_cmd_dma_sg(
	struct easelcomm_service *service, char *command_args,
	size_t command_arg_len)
{
	struct easelcomm_dma_sg_header *sg_header;
	void *cmd_sg;
	struct easelcomm_message_metadata *msg_metadata;

	if (WARN_ON(command_arg_len < sizeof(struct easelcomm_dma_sg_header)))
		return;

	sg_header = (struct easelcomm_dma_sg_header *) command_args;
	command_args += sizeof(struct easelcomm_dma_sg_header);
	command_arg_len -= sizeof(struct easelcomm_dma_sg_header);

	if (WARN_ON(command_arg_len < sg_header->scatterlist_size))
		return;

	if (sg_header->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT) {
		msg_metadata =
			easelcomm_find_local_message(
				service, sg_header->message_id);
	} else {
		msg_metadata =
			easelcomm_find_remote_message(
				service, sg_header->message_id);
	}

	if (!msg_metadata) {
		dev_err(easelcomm_miscdev.this_device,
			"DMA_SG msg %u:%s%llu not found\n",
			service->service_id,
			sg_header->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
			"l" : "r", sg_header->message_id);
		return;
	}

	dev_dbg(easelcomm_miscdev.this_device,
		"recv cmd DMA_SG msg %u:%s%llu size=%u\n",
		service->service_id,
		easelcomm_msgid_prefix(msg_metadata), sg_header->message_id,
		sg_header->scatterlist_size);

	if (WARN_ON(msg_metadata->dma_xfer.sg_remote))
		goto out;

	msg_metadata->dma_xfer.sg_remote_size = sg_header->scatterlist_size;

	if (sg_header->scatterlist_size) {
		cmd_sg = command_args;
		msg_metadata->dma_xfer.sg_remote =
			kmalloc(sg_header->scatterlist_size, GFP_KERNEL);
		if (WARN_ON(!msg_metadata->dma_xfer.sg_remote)) {
			msg_metadata->dma_xfer.sg_remote_size = 0;
			goto out;
		}

		memcpy(msg_metadata->dma_xfer.sg_remote, cmd_sg,
			sg_header->scatterlist_size);
	}

	/* Let the local waiter know the remote scatterlist ready. */
	complete(&msg_metadata->dma_xfer.sg_remote_ready);

out:
	easelcomm_drop_reference(service, msg_metadata, false);
}
EXPORT_SYMBOL(easelcomm_handle_cmd_dma_sg);

/*
 * Client receives DMA transfer instructions from server.  Server has chosen
 * single- vs. multi-block as necessary, and has provided either the single
 * block server-side source or destination address, or the multi-block
 * linked-list address.  Wake up the local process to perform the DMA
 * transfer.
 */
void easelcomm_handle_cmd_dma_xfer(
	struct easelcomm_service *service, char *command_args,
	size_t command_arg_len)
{
	struct easelcomm_dma_xfer_arg *dma_xfer;
	struct easelcomm_message_metadata *msg_metadata;

	if (WARN_ON(command_arg_len <
			sizeof(struct easelcomm_dma_xfer_arg)))
		return;

	dma_xfer = (struct easelcomm_dma_xfer_arg *) command_args;

	dev_dbg(easelcomm_miscdev.this_device,
		"recv cmd DMA_XFER msg %u:%s%llu type=%u saddr=%llx\n",
		service->service_id,
		dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER ? "l" : "r",
		dma_xfer->message_id, dma_xfer->xfer_type,
		dma_xfer->server_addr);

	if (dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER) {
		msg_metadata =
			easelcomm_find_local_message(
				service, dma_xfer->message_id);
	} else {
		msg_metadata =
			easelcomm_find_remote_message(
				service, dma_xfer->message_id);
	}

	if (!msg_metadata) {
		dev_err(easelcomm_miscdev.this_device,
			"DMA_XFER msg %u:%s%llu not found\n",
			service->service_id,
			dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER ?
			"l" : "r", dma_xfer->message_id);
		return;
	}

	msg_metadata->dma_xfer.xfer_type = dma_xfer->xfer_type;
	msg_metadata->dma_xfer.server_addr = dma_xfer->server_addr;

	/* Let the local waiter know the DMA request is received */
	complete(&msg_metadata->dma_xfer.xfer_ready);
	easelcomm_drop_reference(service, msg_metadata, false);
}
EXPORT_SYMBOL(easelcomm_handle_cmd_dma_xfer);

/*
 * Server receives DMA done indication from client.
 * Safe to unmap buffers and return to waiting DMA originator or receiver.
 */
void easelcomm_handle_cmd_dma_done(
	struct easelcomm_service *service, char *command_args,
	size_t command_arg_len)
{
	struct easelcomm_dma_done_arg *dma_done_arg;
	struct easelcomm_message_metadata *msg_metadata;

	if (WARN_ON(command_arg_len < sizeof(struct easelcomm_dma_done_arg)))
		return;

	dma_done_arg = (struct easelcomm_dma_done_arg *) command_args;
	dev_dbg(easelcomm_miscdev.this_device,
		"recv cmd DMA_DONE msg %u:%s%llu err=%u\n",
		service->service_id,
		dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
		"l" : "r", dma_done_arg->message_id, dma_done_arg->errcode);

	if (dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT) {
		msg_metadata =
			easelcomm_find_local_message(
				service, dma_done_arg->message_id);
	} else {
		msg_metadata =
			easelcomm_find_remote_message(
				service, dma_done_arg->message_id);
	}

	if (!msg_metadata) {
		dev_err(easelcomm_miscdev.this_device,
			"CMD_DMA_DONE msg %u:%s%llu not found\n",
			service->service_id,
			dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
			"l" : "r", dma_done_arg->message_id);
		return;
	}

	msg_metadata->dma_xfer.errcode = dma_done_arg->errcode;
	/*
	 * Wakeup local waiter waiting on remote SG (if this is an abort) or
	 * DMA completion.
	 */
	complete(&msg_metadata->dma_xfer.sg_remote_ready);
	complete(&msg_metadata->dma_xfer.xfer_done);
	easelcomm_drop_reference(service, msg_metadata, false);
}
EXPORT_SYMBOL(easelcomm_handle_cmd_dma_done);

static void *easelcomm_create_dma_scatterlist(
	struct easelcomm_kbuf_desc *buf_desc, uint32_t *scatterlist_size,
	void **sglocaldata, enum easelcomm_dma_direction dma_dir)
{
	return easelcomm_hw_build_scatterlist(buf_desc, scatterlist_size,
		sglocaldata, dma_dir);
}

/*
 * Client sends its local DMA dest scatter-gather list to the server, which
 * will use this info to choose a single- or multi-block transfer, and build
 * an MNH DMA Linked List structure based on the local and remote
 * scatter-gather lists, if needed.
 *
 * If the client is discarding the DMA transfer then a zero-length
 * scatter-gather list is sent.
 */
static int easelcomm_send_dma_scatterlist(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dma_dir)
{
	struct easelcomm_dma_sg_header sg_header;
	int ret;

	sg_header.message_id = msg_metadata->msg->desc.message_id;
	sg_header.dma_dir = dma_dir;
	sg_header.scatterlist_size = msg_metadata->dma_xfer.sg_local_size;

	dev_dbg(easelcomm_miscdev.this_device,
		"send cmd DMA_SG msg %u:%s%llu size=%u\n",
		service->service_id, easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id,
		msg_metadata->dma_xfer.sg_local_size);
	ret = easelcomm_start_cmd(
		service, EASELCOMM_CMD_DMA_SG,
		sizeof(struct easelcomm_dma_sg_header) +
		msg_metadata->dma_xfer.sg_local_size);
	if (ret)
		return ret;

	ret = easelcomm_append_cmd_args(
		service, &sg_header, sizeof(struct easelcomm_dma_sg_header));
	if (ret)
		return ret;

	/* If not discarding locally append the scatterlist */
	if (msg_metadata->dma_xfer.sg_local_size) {
		ret = easelcomm_append_cmd_args(
			service, msg_metadata->dma_xfer.sg_local,
			msg_metadata->dma_xfer.sg_local_size);
		if (ret)
			return ret;
	}

	return easelcomm_send_cmd(service);
}

/* Server sends DMA_XFER command to client */
static int easelcomm_send_dma_xfer(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dma_dir)
{
	struct easelcomm_dma_xfer_arg dma_xfer;
	int ret;

	dma_xfer.message_id = msg_metadata->msg->desc.message_id;
	dma_xfer.dma_dir = dma_dir;
	dma_xfer.xfer_type = msg_metadata->dma_xfer.xfer_type;
	dma_xfer.server_addr = msg_metadata->dma_xfer.server_addr;

	ret = easelcomm_start_cmd(
		service, EASELCOMM_CMD_DMA_XFER,
		sizeof(struct easelcomm_dma_xfer_arg));
	if (ret)
		return ret;

	ret = easelcomm_append_cmd_args(
		service, &dma_xfer, sizeof(struct easelcomm_dma_xfer_arg));
	if (ret)
		return ret;
	dev_dbg(easelcomm_miscdev.this_device, "send cmd DMA_XFER msg %u:%s%llu type=%u saddr=%llx\n",
		service->service_id,
		easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id,
		msg_metadata->dma_xfer.xfer_type,
		msg_metadata->dma_xfer.server_addr);
	return easelcomm_send_cmd(service);
}

/*
 * Server-side DMA handling.  When the client sends its scatterlist, inspect
 * the client and server lists to see if the DMA can be a single-block
 * transfer or whether it must be a multi-block transfer.  Construct the
 * multi-block linked list if needed.  Tell the client to proceed with a
 * single-block or multi-block transfer, wait for DMA done command from
 * client, and clean up.
 */
static int easelcomm_server_handle_dma_request(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dma_dir)
{
	void *mblk_ll_data = NULL;
	bool discard = false;
	int ret;

	/* Wait for client to send its scatterlist, if not already */
	ret = wait_for_completion_interruptible(
		&msg_metadata->dma_xfer.sg_remote_ready);
	if (ret || msg_metadata->dma_xfer.aborting)
		goto abort;

	if (!msg_metadata->dma_xfer.sg_remote_size) {
		dev_dbg(easelcomm_miscdev.this_device,
			"client discards DMA for msg %u:%s%llu\n",
			service->service_id,
			easelcomm_msgid_prefix(msg_metadata),
			msg_metadata->msg->desc.message_id);
		return 0;
	}

	/* If discarding locally then send an abort to client. */
	if (msg_metadata->dma_xfer.sg_local_size == 0) {
		discard = true;
		goto abort;
	}

	/*
	 * Choose SBLK vs. MBLK DMA based on whether scatterlists need more
	 * than one block.
	 */
	if (easelcomm_hw_scatterlist_block_count(
			msg_metadata->dma_xfer.sg_local_size) == 1 &&
		easelcomm_hw_scatterlist_block_count(
			msg_metadata->dma_xfer.sg_remote_size) == 1) {
		/*
		 * Single-block DMA.  server_addr for the DMA_XFER command
		 * is the Easel-side starting physical address.
		 */
		msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_SBLK;
		msg_metadata->dma_xfer.server_addr =
			easelcomm_hw_scatterlist_sblk_addr(
				msg_metadata->dma_xfer.sg_local);
	} else {
		/*
		 * Multiple-block DMA.	server_addr is the physical address
		 * of the Linked List we create here.
		 */
		msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_MBLK;

		/* Build DMA Linked List for the transfer*/
		if (dma_dir == EASELCOMM_DMA_DIR_TO_SERVER)
			ret = easelcomm_hw_easel_build_ll(
				msg_metadata->dma_xfer.sg_remote,
				msg_metadata->dma_xfer.sg_local,
				&mblk_ll_data);
		else
			ret = easelcomm_hw_easel_build_ll(
				msg_metadata->dma_xfer.sg_local,
				msg_metadata->dma_xfer.sg_remote,
				&mblk_ll_data);

		/* If error, tell client to abort DMA transfer */
		if (ret < 0 || !mblk_ll_data)
			goto abort;
		msg_metadata->dma_xfer.server_addr =
			easelcomm_hw_easel_ll_addr(mblk_ll_data);
	}

	/* Send DMA_XFER to client */
	ret = easelcomm_send_dma_xfer(service, msg_metadata, dma_dir);
	if (ret)
		goto abort;

	/* Wait for client to send a DMA Done for this message */
	ret = wait_for_completion_interruptible(
		&msg_metadata->dma_xfer.xfer_done);
	if (ret || msg_metadata->dma_xfer.aborting)
		goto abort;

	ret = msg_metadata->dma_xfer.errcode;
	goto out;

abort:
	dev_dbg(easelcomm_miscdev.this_device,
		"aborting DMA for msg %u:%s%llu\n",
		service->service_id, easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id);
	msg_metadata->dma_xfer.aborting = true;
	/* Send a DMA abort to remote */
	msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_ABORT;
	ret = easelcomm_send_dma_xfer(service, msg_metadata, dma_dir);
	if (ret)
		dev_err(easelcomm_miscdev.this_device,
			"send DMA abort for msg %u:%s%llu failed: %d\n",
			service->service_id,
			easelcomm_msgid_prefix(msg_metadata),
			msg_metadata->msg->desc.message_id, ret);
	ret = discard ? 0 : -EIO;

	/*
	 * TODO-LATER: Need to make sure DMA hardware no longer refers to
	 * LL before destroy.  Related: Ensure DMA hardware no longer
	 * accessing the locally-pinned memory pages prior to SG unmap.
	 */

out:
	/* Destroy LL if allocated for MBLK DMA */
	if (mblk_ll_data)
		easelcomm_hw_easel_destroy_ll(mblk_ll_data);
	return ret;
}

/* Client sends DMA Done command to server to let it know it can clean up. */
static int easelcomm_send_dma_done(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dma_dir, int32_t errcode)
{
	struct easelcomm_dma_done_arg dma_done_arg;
	int ret = 0;

	/* Send DMA Done command to server */
	dma_done_arg.message_id =
		msg_metadata->msg->desc.message_id;
	dma_done_arg.dma_dir = dma_dir;
	dma_done_arg.errcode = errcode;
	ret = easelcomm_start_cmd(
		service, EASELCOMM_CMD_DMA_DONE,
		sizeof(struct easelcomm_dma_done_arg));
	if (ret)
		return ret;
	ret = easelcomm_append_cmd_args(
		service, &dma_done_arg, sizeof(struct easelcomm_dma_done_arg));
	if (ret)
		return ret;
	dev_dbg(easelcomm_miscdev.this_device,
		"send cmd DMA_DONE msg %u:%s%llu errcode=%d\n",
		service->service_id, easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id, errcode);
	ret = easelcomm_send_cmd(service);
	return ret;
}

/* Client performs single-block DMA transfer */
static int easelcomm_client_perform_dma_sblk(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dir)
{
	uint64_t client_addr = easelcomm_hw_scatterlist_sblk_addr(
		msg_metadata->dma_xfer.sg_local);

	dev_dbg(easelcomm_miscdev.this_device, "sblk dma msg %u:%s%llu dir=%d size=%u local=%llx saddr=%llx\n",
		service->service_id, easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id, dir,
		msg_metadata->msg->desc.dma_buf_size, client_addr,
		msg_metadata->dma_xfer.server_addr);
	return easelcomm_hw_ap_dma_sblk_transfer(
		client_addr, msg_metadata->dma_xfer.server_addr,
		msg_metadata->msg->desc.dma_buf_size,
		dir == EASELCOMM_DMA_DIR_TO_SERVER);
}

/* Client performs multi-block DMA transfer */
static int easelcomm_client_perform_dma_mblk(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dir)
{
	dev_dbg(easelcomm_miscdev.this_device, "mblk dma msg %u:%s%llu dir=%d size=%u ll=%llx\n",
		service->service_id, easelcomm_msgid_prefix(msg_metadata),
		msg_metadata->msg->desc.message_id, dir,
		msg_metadata->msg->desc.dma_buf_size,
		msg_metadata->dma_xfer.server_addr);
	return easelcomm_hw_ap_dma_mblk_transfer(
		msg_metadata->dma_xfer.server_addr,
		dir == EASELCOMM_DMA_DIR_TO_SERVER);
}

/*
 * Client handles DMA transfer.  Send local scatterlist to server, wait for
 * instructions back on how to proceed with single- or multi-block transfer,
 * perform the transfer, and send the DMA done command to server.
 */
static int easelcomm_client_handle_dma_request(
	struct easelcomm_service *service,
	struct easelcomm_message_metadata *msg_metadata,
	enum easelcomm_dma_direction dma_dir,
	int timeout_ms)
{
	int ret, ret2;

	/* Send scatterlist to server in a DMA_SG command */
	ret = easelcomm_send_dma_scatterlist(
		service, msg_metadata, dma_dir);
	if (ret)
		return ret;

	/* If sent a DMA discard then done */
	if (!msg_metadata->dma_xfer.sg_local_size)
		return 0;

	/* Wait for server to return the DMA request info */
	ret = wait_for_completion_interruptible_timeout(
		&msg_metadata->dma_xfer.xfer_ready,
		msecs_to_jiffies(timeout_ms));
	if (ret == -ERESTARTSYS) {
		dev_info(easelcomm_miscdev.this_device,
				"Wait for DMA_XFER from server interrupted\n");
		goto abort;
	} else if (ret == 0) {
		dev_err(easelcomm_miscdev.this_device,
			"Wait for DMA_XFER from server timed out\n");
		goto abort;
	} else if (msg_metadata->dma_xfer.aborting)
		goto abort;

	/*
	 * If server discards/aborts the DMA request then done.  If
	 * this is a client DMA send then silently discard.  If the
	 * client is receiving a DMA buffer then return an error on
	 * the read.
	 */
	if (msg_metadata->dma_xfer.xfer_type == EASELCOMM_DMA_XFER_ABORT)
		return dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ? -EIO : 0;

	/* Sanity check that list is properly terminated */
	if (easelcomm_hw_verify_scatterlist(&msg_metadata->dma_xfer)) {
		dev_err(easelcomm_miscdev.this_device,
			"sg fails verification\n");
		goto abort;
	}

	/* Single-Block or Multi-Block DMA? */
	if (msg_metadata->dma_xfer.xfer_type == EASELCOMM_DMA_XFER_SBLK) {
		/* Perform the SBLK DMA transfer */
		ret = easelcomm_client_perform_dma_sblk(
			service, msg_metadata, dma_dir);
	} else {
		/* Perform the MBLK DMA transfer */
		ret = easelcomm_client_perform_dma_mblk(
			service, msg_metadata, dma_dir);
	}

	/* Tell server DMA transfer is done */
	ret2 = easelcomm_send_dma_done(service, msg_metadata, dma_dir, ret);
	return ret || ret2;

	/*
	 * We are aborting the transfer before it started (local flush or
	 * signal received)
	 */
abort:
	msg_metadata->dma_xfer.aborting = true;
	/* send abort to server */
	msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_ABORT;
	easelcomm_send_dma_done(service, msg_metadata, dma_dir, -EIO);
	return -EIO;
}

/* RECVDMA ioctl processing. */
int easelcomm_receive_dma(
	struct easelcomm_service *service,
	struct easelcomm_kbuf_desc *buf_desc)
{
	struct easelcomm_message_metadata *msg_metadata;
	enum easelcomm_dma_direction dma_dir;
	int ret = 0;

	dev_dbg(easelcomm_miscdev.this_device,
			"RECVDMA msg %u:r%llu buf_type=%d buf_size=%u timeout_ms=%d dma_buf_fd=%d dma_buf_off=%u dma_buf_width=%u dma_buf_stride=%u\n",
			service->service_id, buf_desc->message_id,
			buf_desc->buf_type, buf_desc->buf_size,
			buf_desc->wait.timeout_ms,
			buf_desc->dma_buf_fd,
			buf_desc->dma_buf_off,
			buf_desc->dma_buf_width,
			buf_desc->dma_buf_stride);

	msg_metadata =
		easelcomm_find_remote_message(service, buf_desc->message_id);
	if (!msg_metadata) {
		dev_err(easelcomm_miscdev.this_device, "RECVDMA msg r%llu not found\n",
			buf_desc->message_id);
		return -EINVAL;
	}

	if (buf_desc->buf_size &&
	    buf_desc->buf_size != msg_metadata->msg->desc.dma_buf_size) {
		dev_err(easelcomm_miscdev.this_device,
			"RECVDMA descriptor buffer size %u doesn't match msg r%llu size %u\n",
			buf_desc->buf_size, buf_desc->message_id,
			msg_metadata->msg->desc.dma_buf_size);
		ret = -EINVAL;
		goto out;
	}

	/* If the message doesn't even request a DMA transfer then skip it. */

	if (!msg_metadata->msg->desc.dma_buf_size)
		goto out;

	/* If it's a user buffer, check valid range and writable. */
	if (buf_desc->buf_type == EASELCOMM_DMA_BUFFER_USER) {
		if (!access_ok(VERIFY_WRITE, buf_desc->buf,
			       buf_desc->buf_size)) {
			ret = -EFAULT;
			goto out;
		}
	}
	dma_dir = easelcomm_is_client() ?
		EASELCOMM_DMA_DIR_TO_CLIENT : EASELCOMM_DMA_DIR_TO_SERVER;

	/*
	 * If the DMA transfer is not being discarded locally then generate
	 * the local DMA scatter-gather list.
	 */
	msg_metadata->dma_xfer.sg_local_size = 0;
	msg_metadata->dma_xfer.sg_local_localdata = NULL;

	if (buf_desc->buf_size) {
		msg_metadata->dma_xfer.sg_local =
			easelcomm_create_dma_scatterlist(
				buf_desc,
				&msg_metadata->dma_xfer.sg_local_size,
				&msg_metadata->dma_xfer.sg_local_localdata,
				dma_dir);
		if (!msg_metadata->dma_xfer.sg_local) {
			ret = -ENOMEM;
			msg_metadata->dma_xfer.sg_local_size = 0;
			goto out;
		}
	}

	if (easelcomm_is_client())
		ret = easelcomm_client_handle_dma_request(
			service, msg_metadata, dma_dir,
			buf_desc->wait.timeout_ms);
	else
		ret = easelcomm_server_handle_dma_request(
			service, msg_metadata, dma_dir);

	/* Free contents of struct mnh_sg_list */
	if (msg_metadata->dma_xfer.sg_local_localdata)
		easelcomm_hw_destroy_scatterlist(
			msg_metadata->dma_xfer.sg_local_localdata);

out:
	/* If no reply needed then done with the remote message. */
	easelcomm_drop_reference(service, msg_metadata,
				!msg_metadata->msg->desc.need_reply);
	return ret;
}
EXPORT_SYMBOL(easelcomm_receive_dma);

/* SENDDMA ioctl processing. */
int easelcomm_send_dma(
	struct easelcomm_service *service,
	struct easelcomm_kbuf_desc *buf_desc)
{
	struct easelcomm_message_metadata *msg_metadata;
	enum easelcomm_dma_direction dma_dir;
	int ret = 0;

	dev_dbg(easelcomm_miscdev.this_device,
			"SENDDMA msg %u:l%llu buf_type=%d buf_size=%u timeout_ms=%d dma_buf_fd=%d dma_buf_off=%u dma_buf_width=%u dma_buf_stride=%u\n",
			service->service_id, buf_desc->message_id,
			buf_desc->buf_type, buf_desc->buf_size,
			buf_desc->wait.timeout_ms,
			buf_desc->dma_buf_fd, buf_desc->dma_buf_off,
			buf_desc->dma_buf_width, buf_desc->dma_buf_stride);

	msg_metadata =
		easelcomm_find_local_message(service, buf_desc->message_id);
	if (!msg_metadata) {
		dev_err(easelcomm_miscdev.this_device, "SENDDMA msg l%llu not found\n",
			buf_desc->message_id);
		return -EINVAL;
	}

	if (buf_desc->buf_size &&
		buf_desc->buf_size != msg_metadata->msg->desc.dma_buf_size) {
		dev_err(easelcomm_miscdev.this_device,
			"SENDDMA descriptor buffer size %u doesn't match message l%llu size %u\n",
			buf_desc->buf_size, buf_desc->message_id,
			msg_metadata->msg->desc.dma_buf_size);
		ret = -EINVAL;
		goto out;
	}

	/* If it's a user buffer, check valid range and readable. */
	if (buf_desc->buf_type == EASELCOMM_DMA_BUFFER_USER) {
		if (!access_ok(VERIFY_READ, buf_desc->buf,
			       buf_desc->buf_size)) {
			ret = -EFAULT;
			goto out;
		}
	}

	dma_dir = easelcomm_is_client() ?
		EASELCOMM_DMA_DIR_TO_SERVER : EASELCOMM_DMA_DIR_TO_CLIENT;

	/*
	 * If the DMA transfer is not being discarded locally then generate
	 * the local DMA scatter-gather list.
	 */
	msg_metadata->dma_xfer.sg_local_size = 0;
	msg_metadata->dma_xfer.sg_local_localdata = NULL;
	if (buf_desc->buf_size) {
		msg_metadata->dma_xfer.sg_local =
			easelcomm_create_dma_scatterlist(
				buf_desc,
				&msg_metadata->dma_xfer.sg_local_size,
				&msg_metadata->dma_xfer.sg_local_localdata,
				dma_dir);
		if (!msg_metadata->dma_xfer.sg_local) {
			ret = -ENOMEM;
			goto out;
		}
	}

	if (msg_metadata->msg->desc.dma_buf_size) {
		if (easelcomm_is_client())
			ret = easelcomm_client_handle_dma_request(
				service, msg_metadata, dma_dir,
				buf_desc->wait.timeout_ms);
		else
			ret = easelcomm_server_handle_dma_request(
				service, msg_metadata, dma_dir);
	}

	/* Free contents of struct mnh_sg_list */
	if (msg_metadata->dma_xfer.sg_local_localdata)
		easelcomm_hw_destroy_scatterlist(
			msg_metadata->dma_xfer.sg_local_localdata);

out:
	/* If no reply needed then done with local message. */
	easelcomm_drop_reference(service, msg_metadata,
		!msg_metadata->msg->desc.need_reply);
	return ret;
}
EXPORT_SYMBOL(easelcomm_send_dma);