google-easel-comm-dma.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751
  1. /*
  2. * Android/Easel coprocessor communication DMA routines.
  3. *
  4. * Copyright 2016 Google Inc.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. /* #define DEBUG */
  11. #include <uapi/linux/google-easel-comm.h>
  12. #include "google-easel-comm-shared.h"
  13. #include "google-easel-comm-private.h"
  14. #include <linux/uaccess.h>
  15. #include <linux/completion.h>
  16. #include <linux/device.h>
  17. #include <linux/kernel.h>
  18. #include <linux/list.h>
  19. #include <linux/miscdevice.h>
  20. #include <linux/module.h>
  21. #include <linux/slab.h>
  22. #include <linux/types.h>
  23. /*
  24. * Server receives DMA scatter-gather list from client. Store this and
  25. * wake up the local process handling the DMA request.
  26. */
  27. void easelcomm_handle_cmd_dma_sg(
  28. struct easelcomm_service *service, char *command_args,
  29. size_t command_arg_len)
  30. {
  31. struct easelcomm_dma_sg_header *sg_header;
  32. void *cmd_sg;
  33. struct easelcomm_message_metadata *msg_metadata;
  34. if (WARN_ON(command_arg_len < sizeof(struct easelcomm_dma_sg_header)))
  35. return;
  36. sg_header = (struct easelcomm_dma_sg_header *) command_args;
  37. command_args += sizeof(struct easelcomm_dma_sg_header);
  38. command_arg_len -= sizeof(struct easelcomm_dma_sg_header);
  39. if (WARN_ON(command_arg_len < sg_header->scatterlist_size))
  40. return;
  41. if (sg_header->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT) {
  42. msg_metadata =
  43. easelcomm_find_local_message(
  44. service, sg_header->message_id);
  45. } else {
  46. msg_metadata =
  47. easelcomm_find_remote_message(
  48. service, sg_header->message_id);
  49. }
  50. if (!msg_metadata) {
  51. dev_err(easelcomm_miscdev.this_device,
  52. "DMA_SG msg %u:%s%llu not found\n",
  53. service->service_id,
  54. sg_header->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
  55. "l" : "r", sg_header->message_id);
  56. return;
  57. }
  58. dev_dbg(easelcomm_miscdev.this_device,
  59. "recv cmd DMA_SG msg %u:%s%llu size=%u\n",
  60. service->service_id,
  61. easelcomm_msgid_prefix(msg_metadata), sg_header->message_id,
  62. sg_header->scatterlist_size);
  63. if (WARN_ON(msg_metadata->dma_xfer.sg_remote))
  64. goto out;
  65. msg_metadata->dma_xfer.sg_remote_size = sg_header->scatterlist_size;
  66. if (sg_header->scatterlist_size) {
  67. cmd_sg = command_args;
  68. msg_metadata->dma_xfer.sg_remote =
  69. kmalloc(sg_header->scatterlist_size, GFP_KERNEL);
  70. if (WARN_ON(!msg_metadata->dma_xfer.sg_remote)) {
  71. msg_metadata->dma_xfer.sg_remote_size = 0;
  72. goto out;
  73. }
  74. memcpy(msg_metadata->dma_xfer.sg_remote, cmd_sg,
  75. sg_header->scatterlist_size);
  76. }
  77. /* Let the local waiter know the remote scatterlist ready. */
  78. complete(&msg_metadata->dma_xfer.sg_remote_ready);
  79. out:
  80. easelcomm_drop_reference(service, msg_metadata, false);
  81. }
  82. EXPORT_SYMBOL(easelcomm_handle_cmd_dma_sg);
  83. /*
  84. * Client receives DMA transfer instructions from server. Server has chosen
  85. * single- vs. multi-block as necessary, and has provided either the single
  86. * block server-side source or destination address, or the multi-block
  87. * linked-list address. Wake up the local process to perform the DMA
  88. * transfer.
  89. */
  90. void easelcomm_handle_cmd_dma_xfer(
  91. struct easelcomm_service *service, char *command_args,
  92. size_t command_arg_len)
  93. {
  94. struct easelcomm_dma_xfer_arg *dma_xfer;
  95. struct easelcomm_message_metadata *msg_metadata;
  96. if (WARN_ON(command_arg_len <
  97. sizeof(struct easelcomm_dma_xfer_arg)))
  98. return;
  99. dma_xfer = (struct easelcomm_dma_xfer_arg *) command_args;
  100. dev_dbg(easelcomm_miscdev.this_device,
  101. "recv cmd DMA_XFER msg %u:%s%llu type=%u saddr=%llx\n",
  102. service->service_id,
  103. dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER ? "l" : "r",
  104. dma_xfer->message_id, dma_xfer->xfer_type,
  105. dma_xfer->server_addr);
  106. if (dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER) {
  107. msg_metadata =
  108. easelcomm_find_local_message(
  109. service, dma_xfer->message_id);
  110. } else {
  111. msg_metadata =
  112. easelcomm_find_remote_message(
  113. service, dma_xfer->message_id);
  114. }
  115. if (!msg_metadata) {
  116. dev_err(easelcomm_miscdev.this_device,
  117. "DMA_XFER msg %u:%s%llu not found\n",
  118. service->service_id,
  119. dma_xfer->dma_dir == EASELCOMM_DMA_DIR_TO_SERVER ?
  120. "l" : "r", dma_xfer->message_id);
  121. return;
  122. }
  123. msg_metadata->dma_xfer.xfer_type = dma_xfer->xfer_type;
  124. msg_metadata->dma_xfer.server_addr = dma_xfer->server_addr;
  125. /* Let the local waiter know the DMA request is received */
  126. complete(&msg_metadata->dma_xfer.xfer_ready);
  127. easelcomm_drop_reference(service, msg_metadata, false);
  128. }
  129. EXPORT_SYMBOL(easelcomm_handle_cmd_dma_xfer);
  130. /*
  131. * Server receives DMA done indication from client.
  132. * Safe to unmap buffers and return to waiting DMA originator or receiver.
  133. */
  134. void easelcomm_handle_cmd_dma_done(
  135. struct easelcomm_service *service, char *command_args,
  136. size_t command_arg_len)
  137. {
  138. struct easelcomm_dma_done_arg *dma_done_arg;
  139. struct easelcomm_message_metadata *msg_metadata;
  140. if (WARN_ON(command_arg_len < sizeof(struct easelcomm_dma_done_arg)))
  141. return;
  142. dma_done_arg = (struct easelcomm_dma_done_arg *) command_args;
  143. dev_dbg(easelcomm_miscdev.this_device,
  144. "recv cmd DMA_DONE msg %u:%s%llu err=%u\n",
  145. service->service_id,
  146. dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
  147. "l" : "r", dma_done_arg->message_id, dma_done_arg->errcode);
  148. if (dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT) {
  149. msg_metadata =
  150. easelcomm_find_local_message(
  151. service, dma_done_arg->message_id);
  152. } else {
  153. msg_metadata =
  154. easelcomm_find_remote_message(
  155. service, dma_done_arg->message_id);
  156. }
  157. if (!msg_metadata) {
  158. dev_err(easelcomm_miscdev.this_device,
  159. "CMD_DMA_DONE msg %u:%s%llu not found\n",
  160. service->service_id,
  161. dma_done_arg->dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ?
  162. "l" : "r", dma_done_arg->message_id);
  163. return;
  164. }
  165. msg_metadata->dma_xfer.errcode = dma_done_arg->errcode;
  166. /*
  167. * Wakeup local waiter waiting on remote SG (if this is an abort) or
  168. * DMA completion.
  169. */
  170. complete(&msg_metadata->dma_xfer.sg_remote_ready);
  171. complete(&msg_metadata->dma_xfer.xfer_done);
  172. easelcomm_drop_reference(service, msg_metadata, false);
  173. }
  174. EXPORT_SYMBOL(easelcomm_handle_cmd_dma_done);
  175. static void *easelcomm_create_dma_scatterlist(
  176. struct easelcomm_kbuf_desc *buf_desc, uint32_t *scatterlist_size,
  177. void **sglocaldata, enum easelcomm_dma_direction dma_dir)
  178. {
  179. return easelcomm_hw_build_scatterlist(buf_desc, scatterlist_size,
  180. sglocaldata, dma_dir);
  181. }
  182. /*
  183. * Client sends its local DMA dest scatter-gather list to the server, which
  184. * will use this info to choose a single- or multi-block transfer, and build
  185. * an MNH DMA Linked List structure based on the local and remote
  186. * scatter-gather lists, if needed.
  187. *
  188. * If the client is discarding the DMA transfer then a zero-length
  189. * scatter-gather list is sent.
  190. */
  191. static int easelcomm_send_dma_scatterlist(
  192. struct easelcomm_service *service,
  193. struct easelcomm_message_metadata *msg_metadata,
  194. enum easelcomm_dma_direction dma_dir)
  195. {
  196. struct easelcomm_dma_sg_header sg_header;
  197. int ret;
  198. sg_header.message_id = msg_metadata->msg->desc.message_id;
  199. sg_header.dma_dir = dma_dir;
  200. sg_header.scatterlist_size = msg_metadata->dma_xfer.sg_local_size;
  201. dev_dbg(easelcomm_miscdev.this_device,
  202. "send cmd DMA_SG msg %u:%s%llu size=%u\n",
  203. service->service_id, easelcomm_msgid_prefix(msg_metadata),
  204. msg_metadata->msg->desc.message_id,
  205. msg_metadata->dma_xfer.sg_local_size);
  206. ret = easelcomm_start_cmd(
  207. service, EASELCOMM_CMD_DMA_SG,
  208. sizeof(struct easelcomm_dma_sg_header) +
  209. msg_metadata->dma_xfer.sg_local_size);
  210. if (ret)
  211. return ret;
  212. ret = easelcomm_append_cmd_args(
  213. service, &sg_header, sizeof(struct easelcomm_dma_sg_header));
  214. if (ret)
  215. return ret;
  216. /* If not discarding locally append the scatterlist */
  217. if (msg_metadata->dma_xfer.sg_local_size) {
  218. ret = easelcomm_append_cmd_args(
  219. service, msg_metadata->dma_xfer.sg_local,
  220. msg_metadata->dma_xfer.sg_local_size);
  221. if (ret)
  222. return ret;
  223. }
  224. return easelcomm_send_cmd(service);
  225. }
  226. /* Server sends DMA_XFER command to client */
  227. static int easelcomm_send_dma_xfer(
  228. struct easelcomm_service *service,
  229. struct easelcomm_message_metadata *msg_metadata,
  230. enum easelcomm_dma_direction dma_dir)
  231. {
  232. struct easelcomm_dma_xfer_arg dma_xfer;
  233. int ret;
  234. dma_xfer.message_id = msg_metadata->msg->desc.message_id;
  235. dma_xfer.dma_dir = dma_dir;
  236. dma_xfer.xfer_type = msg_metadata->dma_xfer.xfer_type;
  237. dma_xfer.server_addr = msg_metadata->dma_xfer.server_addr;
  238. ret = easelcomm_start_cmd(
  239. service, EASELCOMM_CMD_DMA_XFER,
  240. sizeof(struct easelcomm_dma_xfer_arg));
  241. if (ret)
  242. return ret;
  243. ret = easelcomm_append_cmd_args(
  244. service, &dma_xfer, sizeof(struct easelcomm_dma_xfer_arg));
  245. if (ret)
  246. return ret;
  247. dev_dbg(easelcomm_miscdev.this_device, "send cmd DMA_XFER msg %u:%s%llu type=%u saddr=%llx\n",
  248. service->service_id,
  249. easelcomm_msgid_prefix(msg_metadata),
  250. msg_metadata->msg->desc.message_id,
  251. msg_metadata->dma_xfer.xfer_type,
  252. msg_metadata->dma_xfer.server_addr);
  253. return easelcomm_send_cmd(service);
  254. }
  255. /*
  256. * Server-side DMA handling. When the client sends its scatterlist, inspect
  257. * the client and server lists to see if the DMA can be a single-block
  258. * transfer or whether it must be a multi-block transfer. Construct the
  259. * multi-block linked list if needed. Tell the client to proceed with a
  260. * single-block or multi-block transfer, wait for DMA done command from
  261. * client, and clean up.
  262. */
  263. static int easelcomm_server_handle_dma_request(
  264. struct easelcomm_service *service,
  265. struct easelcomm_message_metadata *msg_metadata,
  266. enum easelcomm_dma_direction dma_dir)
  267. {
  268. void *mblk_ll_data = NULL;
  269. bool discard = false;
  270. int ret;
  271. /* Wait for client to send its scatterlist, if not already */
  272. ret = wait_for_completion_interruptible(
  273. &msg_metadata->dma_xfer.sg_remote_ready);
  274. if (ret || msg_metadata->dma_xfer.aborting)
  275. goto abort;
  276. if (!msg_metadata->dma_xfer.sg_remote_size) {
  277. dev_dbg(easelcomm_miscdev.this_device,
  278. "client discards DMA for msg %u:%s%llu\n",
  279. service->service_id,
  280. easelcomm_msgid_prefix(msg_metadata),
  281. msg_metadata->msg->desc.message_id);
  282. return 0;
  283. }
  284. /* If discarding locally then send an abort to client. */
  285. if (msg_metadata->dma_xfer.sg_local_size == 0) {
  286. discard = true;
  287. goto abort;
  288. }
  289. /*
  290. * Choose SBLK vs. MBLK DMA based on whether scatterlists need more
  291. * than one block.
  292. */
  293. if (easelcomm_hw_scatterlist_block_count(
  294. msg_metadata->dma_xfer.sg_local_size) == 1 &&
  295. easelcomm_hw_scatterlist_block_count(
  296. msg_metadata->dma_xfer.sg_remote_size) == 1) {
  297. /*
  298. * Single-block DMA. server_addr for the DMA_XFER command
  299. * is the Easel-side starting physical address.
  300. */
  301. msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_SBLK;
  302. msg_metadata->dma_xfer.server_addr =
  303. easelcomm_hw_scatterlist_sblk_addr(
  304. msg_metadata->dma_xfer.sg_local);
  305. } else {
  306. /*
  307. * Multiple-block DMA. server_addr is the physical address
  308. * of the Linked List we create here.
  309. */
  310. msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_MBLK;
  311. /* Build DMA Linked List for the transfer*/
  312. if (dma_dir == EASELCOMM_DMA_DIR_TO_SERVER)
  313. ret = easelcomm_hw_easel_build_ll(
  314. msg_metadata->dma_xfer.sg_remote,
  315. msg_metadata->dma_xfer.sg_local,
  316. &mblk_ll_data);
  317. else
  318. ret = easelcomm_hw_easel_build_ll(
  319. msg_metadata->dma_xfer.sg_local,
  320. msg_metadata->dma_xfer.sg_remote,
  321. &mblk_ll_data);
  322. /* If error, tell client to abort DMA transfer */
  323. if (ret < 0 || !mblk_ll_data)
  324. goto abort;
  325. msg_metadata->dma_xfer.server_addr =
  326. easelcomm_hw_easel_ll_addr(mblk_ll_data);
  327. }
  328. /* Send DMA_XFER to client */
  329. ret = easelcomm_send_dma_xfer(service, msg_metadata, dma_dir);
  330. if (ret)
  331. goto abort;
  332. /* Wait for client to send a DMA Done for this message */
  333. ret = wait_for_completion_interruptible(
  334. &msg_metadata->dma_xfer.xfer_done);
  335. if (ret || msg_metadata->dma_xfer.aborting)
  336. goto abort;
  337. ret = msg_metadata->dma_xfer.errcode;
  338. goto out;
  339. abort:
  340. dev_dbg(easelcomm_miscdev.this_device,
  341. "aborting DMA for msg %u:%s%llu\n",
  342. service->service_id, easelcomm_msgid_prefix(msg_metadata),
  343. msg_metadata->msg->desc.message_id);
  344. msg_metadata->dma_xfer.aborting = true;
  345. /* Send a DMA abort to remote */
  346. msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_ABORT;
  347. ret = easelcomm_send_dma_xfer(service, msg_metadata, dma_dir);
  348. if (ret)
  349. dev_err(easelcomm_miscdev.this_device,
  350. "send DMA abort for msg %u:%s%llu failed: %d\n",
  351. service->service_id,
  352. easelcomm_msgid_prefix(msg_metadata),
  353. msg_metadata->msg->desc.message_id, ret);
  354. ret = discard ? 0 : -EIO;
  355. /*
  356. * TODO-LATER: Need to make sure DMA hardware no longer refers to
  357. * LL before destroy. Related: Ensure DMA hardware no longer
  358. * accessing the locally-pinned memory pages prior to SG unmap.
  359. */
  360. out:
  361. /* Destroy LL if allocated for MBLK DMA */
  362. if (mblk_ll_data)
  363. easelcomm_hw_easel_destroy_ll(mblk_ll_data);
  364. return ret;
  365. }
  366. /* Client sends DMA Done command to server to let it know it can clean up. */
  367. static int easelcomm_send_dma_done(
  368. struct easelcomm_service *service,
  369. struct easelcomm_message_metadata *msg_metadata,
  370. enum easelcomm_dma_direction dma_dir, int32_t errcode)
  371. {
  372. struct easelcomm_dma_done_arg dma_done_arg;
  373. int ret = 0;
  374. /* Send DMA Done command to server */
  375. dma_done_arg.message_id =
  376. msg_metadata->msg->desc.message_id;
  377. dma_done_arg.dma_dir = dma_dir;
  378. dma_done_arg.errcode = errcode;
  379. ret = easelcomm_start_cmd(
  380. service, EASELCOMM_CMD_DMA_DONE,
  381. sizeof(struct easelcomm_dma_done_arg));
  382. if (ret)
  383. return ret;
  384. ret = easelcomm_append_cmd_args(
  385. service, &dma_done_arg, sizeof(struct easelcomm_dma_done_arg));
  386. if (ret)
  387. return ret;
  388. dev_dbg(easelcomm_miscdev.this_device,
  389. "send cmd DMA_DONE msg %u:%s%llu errcode=%d\n",
  390. service->service_id, easelcomm_msgid_prefix(msg_metadata),
  391. msg_metadata->msg->desc.message_id, errcode);
  392. ret = easelcomm_send_cmd(service);
  393. return ret;
  394. }
  395. /* Client performs single-block DMA transfer */
  396. static int easelcomm_client_perform_dma_sblk(
  397. struct easelcomm_service *service,
  398. struct easelcomm_message_metadata *msg_metadata,
  399. enum easelcomm_dma_direction dir)
  400. {
  401. uint64_t client_addr = easelcomm_hw_scatterlist_sblk_addr(
  402. msg_metadata->dma_xfer.sg_local);
  403. dev_dbg(easelcomm_miscdev.this_device, "sblk dma msg %u:%s%llu dir=%d size=%u local=%llx saddr=%llx\n",
  404. service->service_id, easelcomm_msgid_prefix(msg_metadata),
  405. msg_metadata->msg->desc.message_id, dir,
  406. msg_metadata->msg->desc.dma_buf_size, client_addr,
  407. msg_metadata->dma_xfer.server_addr);
  408. return easelcomm_hw_ap_dma_sblk_transfer(
  409. client_addr, msg_metadata->dma_xfer.server_addr,
  410. msg_metadata->msg->desc.dma_buf_size,
  411. dir == EASELCOMM_DMA_DIR_TO_SERVER);
  412. }
  413. /* Client performs multi-block DMA transfer */
  414. static int easelcomm_client_perform_dma_mblk(
  415. struct easelcomm_service *service,
  416. struct easelcomm_message_metadata *msg_metadata,
  417. enum easelcomm_dma_direction dir)
  418. {
  419. dev_dbg(easelcomm_miscdev.this_device, "mblk dma msg %u:%s%llu dir=%d size=%u ll=%llx\n",
  420. service->service_id, easelcomm_msgid_prefix(msg_metadata),
  421. msg_metadata->msg->desc.message_id, dir,
  422. msg_metadata->msg->desc.dma_buf_size,
  423. msg_metadata->dma_xfer.server_addr);
  424. return easelcomm_hw_ap_dma_mblk_transfer(
  425. msg_metadata->dma_xfer.server_addr,
  426. dir == EASELCOMM_DMA_DIR_TO_SERVER);
  427. }
  428. /*
  429. * Client handles DMA transfer. Send local scatterlist to server, wait for
  430. * instructions back on how to proceed with single- or multi-block transfer,
  431. * perform the transfer, and send the DMA done command to server.
  432. */
  433. static int easelcomm_client_handle_dma_request(
  434. struct easelcomm_service *service,
  435. struct easelcomm_message_metadata *msg_metadata,
  436. enum easelcomm_dma_direction dma_dir,
  437. int timeout_ms)
  438. {
  439. int ret, ret2;
  440. /* Send scatterlist to server in a DMA_SG command */
  441. ret = easelcomm_send_dma_scatterlist(
  442. service, msg_metadata, dma_dir);
  443. if (ret)
  444. return ret;
  445. /* If sent a DMA discard then done */
  446. if (!msg_metadata->dma_xfer.sg_local_size)
  447. return 0;
  448. /* Wait for server to return the DMA request info */
  449. ret = wait_for_completion_interruptible_timeout(
  450. &msg_metadata->dma_xfer.xfer_ready,
  451. msecs_to_jiffies(timeout_ms));
  452. if (ret == -ERESTARTSYS) {
  453. dev_info(easelcomm_miscdev.this_device,
  454. "Wait for DMA_XFER from server interrupted\n");
  455. goto abort;
  456. } else if (ret == 0) {
  457. dev_err(easelcomm_miscdev.this_device,
  458. "Wait for DMA_XFER from server timed out\n");
  459. goto abort;
  460. } else if (msg_metadata->dma_xfer.aborting)
  461. goto abort;
  462. /*
  463. * If server discards/aborts the DMA request then done. If
  464. * this is a client DMA send then silently discard. If the
  465. * client is receiving a DMA buffer then return an error on
  466. * the read.
  467. */
  468. if (msg_metadata->dma_xfer.xfer_type == EASELCOMM_DMA_XFER_ABORT)
  469. return dma_dir == EASELCOMM_DMA_DIR_TO_CLIENT ? -EIO : 0;
  470. /* Sanity check that list is properly terminated */
  471. if (easelcomm_hw_verify_scatterlist(&msg_metadata->dma_xfer)) {
  472. dev_err(easelcomm_miscdev.this_device,
  473. "sg fails verification\n");
  474. goto abort;
  475. }
  476. /* Single-Block or Multi-Block DMA? */
  477. if (msg_metadata->dma_xfer.xfer_type == EASELCOMM_DMA_XFER_SBLK) {
  478. /* Perform the SBLK DMA transfer */
  479. ret = easelcomm_client_perform_dma_sblk(
  480. service, msg_metadata, dma_dir);
  481. } else {
  482. /* Perform the MBLK DMA transfer */
  483. ret = easelcomm_client_perform_dma_mblk(
  484. service, msg_metadata, dma_dir);
  485. }
  486. /* Tell server DMA transfer is done */
  487. ret2 = easelcomm_send_dma_done(service, msg_metadata, dma_dir, ret);
  488. return ret || ret2;
  489. /*
  490. * We are aborting the transfer before it started (local flush or
  491. * signal received)
  492. */
  493. abort:
  494. msg_metadata->dma_xfer.aborting = true;
  495. /* send abort to server */
  496. msg_metadata->dma_xfer.xfer_type = EASELCOMM_DMA_XFER_ABORT;
  497. easelcomm_send_dma_done(service, msg_metadata, dma_dir, -EIO);
  498. return -EIO;
  499. }
  500. /* RECVDMA ioctl processing. */
  501. int easelcomm_receive_dma(
  502. struct easelcomm_service *service,
  503. struct easelcomm_kbuf_desc *buf_desc)
  504. {
  505. struct easelcomm_message_metadata *msg_metadata;
  506. enum easelcomm_dma_direction dma_dir;
  507. int ret = 0;
  508. dev_dbg(easelcomm_miscdev.this_device,
  509. "RECVDMA msg %u:r%llu buf_type=%d buf_size=%u timeout_ms=%d dma_buf_fd=%d dma_buf_off=%u dma_buf_width=%u dma_buf_stride=%u\n",
  510. service->service_id, buf_desc->message_id,
  511. buf_desc->buf_type, buf_desc->buf_size,
  512. buf_desc->wait.timeout_ms,
  513. buf_desc->dma_buf_fd,
  514. buf_desc->dma_buf_off,
  515. buf_desc->dma_buf_width,
  516. buf_desc->dma_buf_stride);
  517. msg_metadata =
  518. easelcomm_find_remote_message(service, buf_desc->message_id);
  519. if (!msg_metadata) {
  520. dev_err(easelcomm_miscdev.this_device, "RECVDMA msg r%llu not found\n",
  521. buf_desc->message_id);
  522. return -EINVAL;
  523. }
  524. if (buf_desc->buf_size &&
  525. buf_desc->buf_size != msg_metadata->msg->desc.dma_buf_size) {
  526. dev_err(easelcomm_miscdev.this_device,
  527. "RECVDMA descriptor buffer size %u doesn't match msg r%llu size %u\n",
  528. buf_desc->buf_size, buf_desc->message_id,
  529. msg_metadata->msg->desc.dma_buf_size);
  530. ret = -EINVAL;
  531. goto out;
  532. }
  533. /* If the message doesn't even request a DMA transfer then skip it. */
  534. if (!msg_metadata->msg->desc.dma_buf_size)
  535. goto out;
  536. /* If it's a user buffer, check valid range and writable. */
  537. if (buf_desc->buf_type == EASELCOMM_DMA_BUFFER_USER) {
  538. if (!access_ok(VERIFY_WRITE, buf_desc->buf,
  539. buf_desc->buf_size)) {
  540. ret = -EFAULT;
  541. goto out;
  542. }
  543. }
  544. dma_dir = easelcomm_is_client() ?
  545. EASELCOMM_DMA_DIR_TO_CLIENT : EASELCOMM_DMA_DIR_TO_SERVER;
  546. /*
  547. * If the DMA transfer is not being discarded locally then generate
  548. * the local DMA scatter-gather list.
  549. */
  550. msg_metadata->dma_xfer.sg_local_size = 0;
  551. msg_metadata->dma_xfer.sg_local_localdata = NULL;
  552. if (buf_desc->buf_size) {
  553. msg_metadata->dma_xfer.sg_local =
  554. easelcomm_create_dma_scatterlist(
  555. buf_desc,
  556. &msg_metadata->dma_xfer.sg_local_size,
  557. &msg_metadata->dma_xfer.sg_local_localdata,
  558. dma_dir);
  559. if (!msg_metadata->dma_xfer.sg_local) {
  560. ret = -ENOMEM;
  561. msg_metadata->dma_xfer.sg_local_size = 0;
  562. goto out;
  563. }
  564. }
  565. if (easelcomm_is_client())
  566. ret = easelcomm_client_handle_dma_request(
  567. service, msg_metadata, dma_dir,
  568. buf_desc->wait.timeout_ms);
  569. else
  570. ret = easelcomm_server_handle_dma_request(
  571. service, msg_metadata, dma_dir);
  572. /* Free contents of struct mnh_sg_list */
  573. if (msg_metadata->dma_xfer.sg_local_localdata)
  574. easelcomm_hw_destroy_scatterlist(
  575. msg_metadata->dma_xfer.sg_local_localdata);
  576. out:
  577. /* If no reply needed then done with the remote message. */
  578. easelcomm_drop_reference(service, msg_metadata,
  579. !msg_metadata->msg->desc.need_reply);
  580. return ret;
  581. }
  582. EXPORT_SYMBOL(easelcomm_receive_dma);
  583. /* SENDDMA ioctl processing. */
  584. int easelcomm_send_dma(
  585. struct easelcomm_service *service,
  586. struct easelcomm_kbuf_desc *buf_desc)
  587. {
  588. struct easelcomm_message_metadata *msg_metadata;
  589. enum easelcomm_dma_direction dma_dir;
  590. int ret = 0;
  591. dev_dbg(easelcomm_miscdev.this_device,
  592. "SENDDMA msg %u:l%llu buf_type=%d buf_size=%u timeout_ms=%d dma_buf_fd=%d dma_buf_off=%u dma_buf_width=%u dma_buf_stride=%u\n",
  593. service->service_id, buf_desc->message_id,
  594. buf_desc->buf_type, buf_desc->buf_size,
  595. buf_desc->wait.timeout_ms,
  596. buf_desc->dma_buf_fd, buf_desc->dma_buf_off,
  597. buf_desc->dma_buf_width, buf_desc->dma_buf_stride);
  598. msg_metadata =
  599. easelcomm_find_local_message(service, buf_desc->message_id);
  600. if (!msg_metadata) {
  601. dev_err(easelcomm_miscdev.this_device, "SENDDMA msg l%llu not found\n",
  602. buf_desc->message_id);
  603. return -EINVAL;
  604. }
  605. if (buf_desc->buf_size &&
  606. buf_desc->buf_size != msg_metadata->msg->desc.dma_buf_size) {
  607. dev_err(easelcomm_miscdev.this_device,
  608. "SENDDMA descriptor buffer size %u doesn't match message l%llu size %u\n",
  609. buf_desc->buf_size, buf_desc->message_id,
  610. msg_metadata->msg->desc.dma_buf_size);
  611. ret = -EINVAL;
  612. goto out;
  613. }
  614. /* If it's a user buffer, check valid range and readable. */
  615. if (buf_desc->buf_type == EASELCOMM_DMA_BUFFER_USER) {
  616. if (!access_ok(VERIFY_READ, buf_desc->buf,
  617. buf_desc->buf_size)) {
  618. ret = -EFAULT;
  619. goto out;
  620. }
  621. }
  622. dma_dir = easelcomm_is_client() ?
  623. EASELCOMM_DMA_DIR_TO_SERVER : EASELCOMM_DMA_DIR_TO_CLIENT;
  624. /*
  625. * If the DMA transfer is not being discarded locally then generate
  626. * the local DMA scatter-gather list.
  627. */
  628. msg_metadata->dma_xfer.sg_local_size = 0;
  629. msg_metadata->dma_xfer.sg_local_localdata = NULL;
  630. if (buf_desc->buf_size) {
  631. msg_metadata->dma_xfer.sg_local =
  632. easelcomm_create_dma_scatterlist(
  633. buf_desc,
  634. &msg_metadata->dma_xfer.sg_local_size,
  635. &msg_metadata->dma_xfer.sg_local_localdata,
  636. dma_dir);
  637. if (!msg_metadata->dma_xfer.sg_local) {
  638. ret = -ENOMEM;
  639. goto out;
  640. }
  641. }
  642. if (msg_metadata->msg->desc.dma_buf_size) {
  643. if (easelcomm_is_client())
  644. ret = easelcomm_client_handle_dma_request(
  645. service, msg_metadata, dma_dir,
  646. buf_desc->wait.timeout_ms);
  647. else
  648. ret = easelcomm_server_handle_dma_request(
  649. service, msg_metadata, dma_dir);
  650. }
  651. /* Free contents of struct mnh_sg_list */
  652. if (msg_metadata->dma_xfer.sg_local_localdata)
  653. easelcomm_hw_destroy_scatterlist(
  654. msg_metadata->dma_xfer.sg_local_localdata);
  655. out:
  656. /* If no reply needed then done with local message. */
  657. easelcomm_drop_reference(service, msg_metadata,
  658. !msg_metadata->msg->desc.need_reply);
  659. return ret;
  660. }
  661. EXPORT_SYMBOL(easelcomm_send_dma);