123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245 |
- #include <linux/spinlock.h>
- #include <linux/module.h>
- #include <linux/idr.h>
- #include <linux/kernel.h>
- #include <linux/timer.h>
- #include <linux/parser.h>
- #include <linux/vmalloc.h>
- #include <linux/uio_driver.h>
- #include <linux/stringify.h>
- #include <linux/bitops.h>
- #include <net/genetlink.h>
- #include <scsi/scsi_common.h>
- #include <scsi/scsi_proto.h>
- #include <target/target_core_base.h>
- #include <target/target_core_fabric.h>
- #include <target/target_core_backend.h>
- #include <linux/target_core_user.h>
- #define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
- #define DATA_BLOCK_BITS 256
- #define DATA_BLOCK_SIZE 4096
- #define CMDR_SIZE (16 * 4096)
- #define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE)
- #define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE)
- static struct device *tcmu_root_device;
- struct tcmu_hba {
- u32 host_id;
- };
- #define TCMU_CONFIG_LEN 256
- struct tcmu_dev {
- struct se_device se_dev;
- char *name;
- struct se_hba *hba;
- #define TCMU_DEV_BIT_OPEN 0
- #define TCMU_DEV_BIT_BROKEN 1
- unsigned long flags;
- struct uio_info uio_info;
- struct tcmu_mailbox *mb_addr;
- size_t dev_size;
- u32 cmdr_size;
- u32 cmdr_last_cleaned;
-
-
- size_t data_off;
- size_t data_size;
- DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
- wait_queue_head_t wait_cmdr;
-
- spinlock_t cmdr_lock;
- struct idr commands;
- spinlock_t commands_lock;
- struct timer_list timeout;
- char dev_config[TCMU_CONFIG_LEN];
- };
- #define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
- #define CMDR_OFF sizeof(struct tcmu_mailbox)
- struct tcmu_cmd {
- struct se_cmd *se_cmd;
- struct tcmu_dev *tcmu_dev;
- uint16_t cmd_id;
-
- DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
- unsigned long deadline;
- #define TCMU_CMD_BIT_EXPIRED 0
- unsigned long flags;
- };
- static struct kmem_cache *tcmu_cmd_cache;
- enum tcmu_multicast_groups {
- TCMU_MCGRP_CONFIG,
- };
- static const struct genl_multicast_group tcmu_mcgrps[] = {
- [TCMU_MCGRP_CONFIG] = { .name = "config", },
- };
- static struct genl_family tcmu_genl_family = {
- .id = GENL_ID_GENERATE,
- .hdrsize = 0,
- .name = "TCM-USER",
- .version = 1,
- .maxattr = TCMU_ATTR_MAX,
- .mcgrps = tcmu_mcgrps,
- .n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
- .netnsok = true,
- };
- static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
- {
- struct se_device *se_dev = se_cmd->se_dev;
- struct tcmu_dev *udev = TCMU_DEV(se_dev);
- struct tcmu_cmd *tcmu_cmd;
- int cmd_id;
- tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_KERNEL);
- if (!tcmu_cmd)
- return NULL;
- tcmu_cmd->se_cmd = se_cmd;
- tcmu_cmd->tcmu_dev = udev;
- tcmu_cmd->deadline = jiffies + msecs_to_jiffies(TCMU_TIME_OUT);
- idr_preload(GFP_KERNEL);
- spin_lock_irq(&udev->commands_lock);
- cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0,
- USHRT_MAX, GFP_NOWAIT);
- spin_unlock_irq(&udev->commands_lock);
- idr_preload_end();
- if (cmd_id < 0) {
- kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
- return NULL;
- }
- tcmu_cmd->cmd_id = cmd_id;
- return tcmu_cmd;
- }
- static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
- {
- unsigned long offset = offset_in_page(vaddr);
- size = round_up(size+offset, PAGE_SIZE);
- vaddr -= offset;
- while (size) {
- flush_dcache_page(virt_to_page(vaddr));
- size -= PAGE_SIZE;
- }
- }
- static inline size_t spc_used(size_t head, size_t tail, size_t size)
- {
- int diff = head - tail;
- if (diff >= 0)
- return diff;
- else
- return size + diff;
- }
- static inline size_t spc_free(size_t head, size_t tail, size_t size)
- {
-
- return (size - spc_used(head, tail, size) - 1);
- }
- static inline size_t head_to_end(size_t head, size_t size)
- {
- return size - head;
- }
- static inline void new_iov(struct iovec **iov, int *iov_cnt,
- struct tcmu_dev *udev)
- {
- struct iovec *iovec;
- if (*iov_cnt != 0)
- (*iov)++;
- (*iov_cnt)++;
- iovec = *iov;
- memset(iovec, 0, sizeof(struct iovec));
- }
- #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
- static inline size_t get_block_offset(struct tcmu_dev *dev,
- int block, int remaining)
- {
- return dev->data_off + block * DATA_BLOCK_SIZE +
- DATA_BLOCK_SIZE - remaining;
- }
- static inline size_t iov_tail(struct tcmu_dev *udev, struct iovec *iov)
- {
- return (size_t)iov->iov_base + iov->iov_len;
- }
- static void alloc_and_scatter_data_area(struct tcmu_dev *udev,
- struct scatterlist *data_sg, unsigned int data_nents,
- struct iovec **iov, int *iov_cnt, bool copy_data)
- {
- int i, block;
- int block_remaining = 0;
- void *from, *to;
- size_t copy_bytes, to_offset;
- struct scatterlist *sg;
- for_each_sg(data_sg, sg, data_nents, i) {
- int sg_remaining = sg->length;
- from = kmap_atomic(sg_page(sg)) + sg->offset;
- while (sg_remaining > 0) {
- if (block_remaining == 0) {
- block = find_first_zero_bit(udev->data_bitmap,
- DATA_BLOCK_BITS);
- block_remaining = DATA_BLOCK_SIZE;
- set_bit(block, udev->data_bitmap);
- }
- copy_bytes = min_t(size_t, sg_remaining,
- block_remaining);
- to_offset = get_block_offset(udev, block,
- block_remaining);
- to = (void *)udev->mb_addr + to_offset;
- if (*iov_cnt != 0 &&
- to_offset == iov_tail(udev, *iov)) {
- (*iov)->iov_len += copy_bytes;
- } else {
- new_iov(iov, iov_cnt, udev);
- (*iov)->iov_base = (void __user *) to_offset;
- (*iov)->iov_len = copy_bytes;
- }
- if (copy_data) {
- memcpy(to, from + sg->length - sg_remaining,
- copy_bytes);
- tcmu_flush_dcache_range(to, copy_bytes);
- }
- sg_remaining -= copy_bytes;
- block_remaining -= copy_bytes;
- }
- kunmap_atomic(from - sg->offset);
- }
- }
- static void free_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd)
- {
- bitmap_xor(udev->data_bitmap, udev->data_bitmap, cmd->data_bitmap,
- DATA_BLOCK_BITS);
- }
- static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
- bool bidi)
- {
- struct se_cmd *se_cmd = cmd->se_cmd;
- int i, block;
- int block_remaining = 0;
- void *from, *to;
- size_t copy_bytes, from_offset;
- struct scatterlist *sg, *data_sg;
- unsigned int data_nents;
- DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS);
- bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS);
- if (!bidi) {
- data_sg = se_cmd->t_data_sg;
- data_nents = se_cmd->t_data_nents;
- } else {
- uint32_t count;
-
- count = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE);
- while (count--) {
- block = find_first_bit(bitmap, DATA_BLOCK_BITS);
- clear_bit(block, bitmap);
- }
- data_sg = se_cmd->t_bidi_data_sg;
- data_nents = se_cmd->t_bidi_data_nents;
- }
- for_each_sg(data_sg, sg, data_nents, i) {
- int sg_remaining = sg->length;
- to = kmap_atomic(sg_page(sg)) + sg->offset;
- while (sg_remaining > 0) {
- if (block_remaining == 0) {
- block = find_first_bit(bitmap,
- DATA_BLOCK_BITS);
- block_remaining = DATA_BLOCK_SIZE;
- clear_bit(block, bitmap);
- }
- copy_bytes = min_t(size_t, sg_remaining,
- block_remaining);
- from_offset = get_block_offset(udev, block,
- block_remaining);
- from = (void *) udev->mb_addr + from_offset;
- tcmu_flush_dcache_range(from, copy_bytes);
- memcpy(to + sg->length - sg_remaining, from,
- copy_bytes);
- sg_remaining -= copy_bytes;
- block_remaining -= copy_bytes;
- }
- kunmap_atomic(to - sg->offset);
- }
- }
- static inline size_t spc_bitmap_free(unsigned long *bitmap)
- {
- return DATA_BLOCK_SIZE * (DATA_BLOCK_BITS -
- bitmap_weight(bitmap, DATA_BLOCK_BITS));
- }
- static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed)
- {
- struct tcmu_mailbox *mb = udev->mb_addr;
- size_t space, cmd_needed;
- u32 cmd_head;
- tcmu_flush_dcache_range(mb, sizeof(*mb));
- cmd_head = mb->cmd_head % udev->cmdr_size;
-
- if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)
- cmd_needed = cmd_size;
- else
- cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);
- space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);
- if (space < cmd_needed) {
- pr_debug("no cmd space: %u %u %u\n", cmd_head,
- udev->cmdr_last_cleaned, udev->cmdr_size);
- return false;
- }
- space = spc_bitmap_free(udev->data_bitmap);
- if (space < data_needed) {
- pr_debug("no data space: only %zu available, but ask for %zu\n",
- space, data_needed);
- return false;
- }
- return true;
- }
- static inline size_t tcmu_cmd_get_data_length(struct tcmu_cmd *tcmu_cmd)
- {
- struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
- size_t data_length = round_up(se_cmd->data_length, DATA_BLOCK_SIZE);
- if (se_cmd->se_cmd_flags & SCF_BIDI) {
- BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
- data_length += round_up(se_cmd->t_bidi_data_sg->length,
- DATA_BLOCK_SIZE);
- }
- return data_length;
- }
- static inline uint32_t tcmu_cmd_get_block_cnt(struct tcmu_cmd *tcmu_cmd)
- {
- size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
- return data_length / DATA_BLOCK_SIZE;
- }
- static sense_reason_t
- tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
- {
- struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
- struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
- size_t base_command_size, command_size;
- struct tcmu_mailbox *mb;
- struct tcmu_cmd_entry *entry;
- struct iovec *iov;
- int iov_cnt;
- uint32_t cmd_head;
- uint64_t cdb_off;
- bool copy_to_data_area;
- size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
- DECLARE_BITMAP(old_bitmap, DATA_BLOCK_BITS);
- if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
- return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-
- base_command_size = max(offsetof(struct tcmu_cmd_entry,
- req.iov[tcmu_cmd_get_block_cnt(tcmu_cmd)]),
- sizeof(struct tcmu_cmd_entry));
- command_size = base_command_size
- + round_up(scsi_command_size(se_cmd->t_task_cdb), TCMU_OP_ALIGN_SIZE);
- WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
- spin_lock_irq(&udev->cmdr_lock);
- mb = udev->mb_addr;
- cmd_head = mb->cmd_head % udev->cmdr_size;
- if ((command_size > (udev->cmdr_size / 2)) ||
- data_length > udev->data_size) {
- pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu "
- "cmd ring/data area\n", command_size, data_length,
- udev->cmdr_size, udev->data_size);
- spin_unlock_irq(&udev->cmdr_lock);
- return TCM_INVALID_CDB_FIELD;
- }
- while (!is_ring_space_avail(udev, command_size, data_length)) {
- int ret;
- DEFINE_WAIT(__wait);
- prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE);
- pr_debug("sleeping for ring space\n");
- spin_unlock_irq(&udev->cmdr_lock);
- ret = schedule_timeout(msecs_to_jiffies(TCMU_TIME_OUT));
- finish_wait(&udev->wait_cmdr, &__wait);
- if (!ret) {
- pr_warn("tcmu: command timed out\n");
- return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
- }
- spin_lock_irq(&udev->cmdr_lock);
-
- cmd_head = mb->cmd_head % udev->cmdr_size;
- }
-
- if (head_to_end(cmd_head, udev->cmdr_size) < command_size) {
- size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
- entry = (void *) mb + CMDR_OFF + cmd_head;
- tcmu_flush_dcache_range(entry, sizeof(*entry));
- tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_PAD);
- tcmu_hdr_set_len(&entry->hdr.len_op, pad_size);
- entry->hdr.cmd_id = 0;
- entry->hdr.kflags = 0;
- entry->hdr.uflags = 0;
- UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
- cmd_head = mb->cmd_head % udev->cmdr_size;
- WARN_ON(cmd_head != 0);
- }
- entry = (void *) mb + CMDR_OFF + cmd_head;
- tcmu_flush_dcache_range(entry, sizeof(*entry));
- tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
- tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
- entry->hdr.cmd_id = tcmu_cmd->cmd_id;
- entry->hdr.kflags = 0;
- entry->hdr.uflags = 0;
- bitmap_copy(old_bitmap, udev->data_bitmap, DATA_BLOCK_BITS);
-
- iov = &entry->req.iov[0];
- iov_cnt = 0;
- copy_to_data_area = (se_cmd->data_direction == DMA_TO_DEVICE
- || se_cmd->se_cmd_flags & SCF_BIDI);
- alloc_and_scatter_data_area(udev, se_cmd->t_data_sg,
- se_cmd->t_data_nents, &iov, &iov_cnt, copy_to_data_area);
- entry->req.iov_cnt = iov_cnt;
- entry->req.iov_dif_cnt = 0;
-
- if (se_cmd->se_cmd_flags & SCF_BIDI) {
- iov_cnt = 0;
- iov++;
- alloc_and_scatter_data_area(udev, se_cmd->t_bidi_data_sg,
- se_cmd->t_bidi_data_nents, &iov, &iov_cnt,
- false);
- entry->req.iov_bidi_cnt = iov_cnt;
- }
-
- bitmap_xor(tcmu_cmd->data_bitmap, old_bitmap, udev->data_bitmap,
- DATA_BLOCK_BITS);
-
- cdb_off = CMDR_OFF + cmd_head + base_command_size;
- memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
- entry->req.cdb_off = cdb_off;
- tcmu_flush_dcache_range(entry, sizeof(*entry));
- UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
- tcmu_flush_dcache_range(mb, sizeof(*mb));
- spin_unlock_irq(&udev->cmdr_lock);
-
- uio_event_notify(&udev->uio_info);
- mod_timer(&udev->timeout,
- round_jiffies_up(jiffies + msecs_to_jiffies(TCMU_TIME_OUT)));
- return TCM_NO_SENSE;
- }
- static sense_reason_t
- tcmu_queue_cmd(struct se_cmd *se_cmd)
- {
- struct se_device *se_dev = se_cmd->se_dev;
- struct tcmu_dev *udev = TCMU_DEV(se_dev);
- struct tcmu_cmd *tcmu_cmd;
- int ret;
- tcmu_cmd = tcmu_alloc_cmd(se_cmd);
- if (!tcmu_cmd)
- return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
- ret = tcmu_queue_cmd_ring(tcmu_cmd);
- if (ret != TCM_NO_SENSE) {
- pr_err("TCMU: Could not queue command\n");
- spin_lock_irq(&udev->commands_lock);
- idr_remove(&udev->commands, tcmu_cmd->cmd_id);
- spin_unlock_irq(&udev->commands_lock);
- kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
- }
- return ret;
- }
- static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
- {
- struct se_cmd *se_cmd = cmd->se_cmd;
- struct tcmu_dev *udev = cmd->tcmu_dev;
- if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
-
- free_data_area(udev, cmd);
- kmem_cache_free(tcmu_cmd_cache, cmd);
- return;
- }
- if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
- free_data_area(udev, cmd);
- pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
- cmd->se_cmd);
- entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
- } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
- memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer,
- se_cmd->scsi_sense_length);
- free_data_area(udev, cmd);
- } else if (se_cmd->se_cmd_flags & SCF_BIDI) {
-
- gather_data_area(udev, cmd, true);
- free_data_area(udev, cmd);
- } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
- gather_data_area(udev, cmd, false);
- free_data_area(udev, cmd);
- } else if (se_cmd->data_direction == DMA_TO_DEVICE) {
- free_data_area(udev, cmd);
- } else if (se_cmd->data_direction != DMA_NONE) {
- pr_warn("TCMU: data direction was %d!\n",
- se_cmd->data_direction);
- }
- target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
- cmd->se_cmd = NULL;
- kmem_cache_free(tcmu_cmd_cache, cmd);
- }
- static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
- {
- struct tcmu_mailbox *mb;
- unsigned long flags;
- int handled = 0;
- if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
- pr_err("ring broken, not handling completions\n");
- return 0;
- }
- spin_lock_irqsave(&udev->cmdr_lock, flags);
- mb = udev->mb_addr;
- tcmu_flush_dcache_range(mb, sizeof(*mb));
- while (udev->cmdr_last_cleaned != ACCESS_ONCE(mb->cmd_tail)) {
- struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;
- struct tcmu_cmd *cmd;
- tcmu_flush_dcache_range(entry, sizeof(*entry));
- if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD) {
- UPDATE_HEAD(udev->cmdr_last_cleaned,
- tcmu_hdr_get_len(entry->hdr.len_op),
- udev->cmdr_size);
- continue;
- }
- WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
- spin_lock(&udev->commands_lock);
- cmd = idr_find(&udev->commands, entry->hdr.cmd_id);
- if (cmd)
- idr_remove(&udev->commands, cmd->cmd_id);
- spin_unlock(&udev->commands_lock);
- if (!cmd) {
- pr_err("cmd_id not found, ring is broken\n");
- set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
- break;
- }
- tcmu_handle_completion(cmd, entry);
- UPDATE_HEAD(udev->cmdr_last_cleaned,
- tcmu_hdr_get_len(entry->hdr.len_op),
- udev->cmdr_size);
- handled++;
- }
- if (mb->cmd_tail == mb->cmd_head)
- del_timer(&udev->timeout);
- spin_unlock_irqrestore(&udev->cmdr_lock, flags);
- wake_up(&udev->wait_cmdr);
- return handled;
- }
- static int tcmu_check_expired_cmd(int id, void *p, void *data)
- {
- struct tcmu_cmd *cmd = p;
- if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
- return 0;
- if (!time_after(jiffies, cmd->deadline))
- return 0;
- set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
- target_complete_cmd(cmd->se_cmd, SAM_STAT_CHECK_CONDITION);
- cmd->se_cmd = NULL;
- return 0;
- }
- static void tcmu_device_timedout(unsigned long data)
- {
- struct tcmu_dev *udev = (struct tcmu_dev *)data;
- unsigned long flags;
- int handled;
- handled = tcmu_handle_completions(udev);
- pr_warn("%d completions handled from timeout\n", handled);
- spin_lock_irqsave(&udev->commands_lock, flags);
- idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);
- spin_unlock_irqrestore(&udev->commands_lock, flags);
-
- }
- static int tcmu_attach_hba(struct se_hba *hba, u32 host_id)
- {
- struct tcmu_hba *tcmu_hba;
- tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL);
- if (!tcmu_hba)
- return -ENOMEM;
- tcmu_hba->host_id = host_id;
- hba->hba_ptr = tcmu_hba;
- return 0;
- }
- static void tcmu_detach_hba(struct se_hba *hba)
- {
- kfree(hba->hba_ptr);
- hba->hba_ptr = NULL;
- }
- static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
- {
- struct tcmu_dev *udev;
- udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);
- if (!udev)
- return NULL;
- udev->name = kstrdup(name, GFP_KERNEL);
- if (!udev->name) {
- kfree(udev);
- return NULL;
- }
- udev->hba = hba;
- init_waitqueue_head(&udev->wait_cmdr);
- spin_lock_init(&udev->cmdr_lock);
- idr_init(&udev->commands);
- spin_lock_init(&udev->commands_lock);
- setup_timer(&udev->timeout, tcmu_device_timedout,
- (unsigned long)udev);
- return &udev->se_dev;
- }
- static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
- {
- struct tcmu_dev *tcmu_dev = container_of(info, struct tcmu_dev, uio_info);
- tcmu_handle_completions(tcmu_dev);
- return 0;
- }
- static int tcmu_find_mem_index(struct vm_area_struct *vma)
- {
- struct tcmu_dev *udev = vma->vm_private_data;
- struct uio_info *info = &udev->uio_info;
- if (vma->vm_pgoff < MAX_UIO_MAPS) {
- if (info->mem[vma->vm_pgoff].size == 0)
- return -1;
- return (int)vma->vm_pgoff;
- }
- return -1;
- }
- static int tcmu_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
- {
- struct tcmu_dev *udev = vma->vm_private_data;
- struct uio_info *info = &udev->uio_info;
- struct page *page;
- unsigned long offset;
- void *addr;
- int mi = tcmu_find_mem_index(vma);
- if (mi < 0)
- return VM_FAULT_SIGBUS;
-
- offset = (vmf->pgoff - mi) << PAGE_SHIFT;
- addr = (void *)(unsigned long)info->mem[mi].addr + offset;
- if (info->mem[mi].memtype == UIO_MEM_LOGICAL)
- page = virt_to_page(addr);
- else
- page = vmalloc_to_page(addr);
- get_page(page);
- vmf->page = page;
- return 0;
- }
- static const struct vm_operations_struct tcmu_vm_ops = {
- .fault = tcmu_vma_fault,
- };
- static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
- {
- struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
- vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
- vma->vm_ops = &tcmu_vm_ops;
- vma->vm_private_data = udev;
-
- if (vma_pages(vma) != (TCMU_RING_SIZE >> PAGE_SHIFT))
- return -EINVAL;
- return 0;
- }
- static int tcmu_open(struct uio_info *info, struct inode *inode)
- {
- struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
-
- if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
- return -EBUSY;
- pr_debug("open\n");
- return 0;
- }
- static int tcmu_release(struct uio_info *info, struct inode *inode)
- {
- struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
- clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);
- pr_debug("close\n");
- return 0;
- }
- static int tcmu_netlink_event(enum tcmu_genl_cmd cmd, const char *name, int minor)
- {
- struct sk_buff *skb;
- void *msg_header;
- int ret = -ENOMEM;
- skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
- if (!skb)
- return ret;
- msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd);
- if (!msg_header)
- goto free_skb;
- ret = nla_put_string(skb, TCMU_ATTR_DEVICE, name);
- if (ret < 0)
- goto free_skb;
- ret = nla_put_u32(skb, TCMU_ATTR_MINOR, minor);
- if (ret < 0)
- goto free_skb;
- genlmsg_end(skb, msg_header);
- ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
- TCMU_MCGRP_CONFIG, GFP_KERNEL);
-
- if (ret == -ESRCH)
- ret = 0;
- return ret;
- free_skb:
- nlmsg_free(skb);
- return ret;
- }
- static int tcmu_configure_device(struct se_device *dev)
- {
- struct tcmu_dev *udev = TCMU_DEV(dev);
- struct tcmu_hba *hba = udev->hba->hba_ptr;
- struct uio_info *info;
- struct tcmu_mailbox *mb;
- size_t size;
- size_t used;
- int ret = 0;
- char *str;
- info = &udev->uio_info;
- size = snprintf(NULL, 0, "tcm-user/%u/%s/%s", hba->host_id, udev->name,
- udev->dev_config);
- size += 1;
- str = kmalloc(size, GFP_KERNEL);
- if (!str)
- return -ENOMEM;
- used = snprintf(str, size, "tcm-user/%u/%s", hba->host_id, udev->name);
- if (udev->dev_config[0])
- snprintf(str + used, size - used, "/%s", udev->dev_config);
- info->name = str;
- udev->mb_addr = vzalloc(TCMU_RING_SIZE);
- if (!udev->mb_addr) {
- ret = -ENOMEM;
- goto err_vzalloc;
- }
-
- udev->cmdr_size = CMDR_SIZE - CMDR_OFF;
- udev->data_off = CMDR_SIZE;
- udev->data_size = TCMU_RING_SIZE - CMDR_SIZE;
- mb = udev->mb_addr;
- mb->version = TCMU_MAILBOX_VERSION;
- mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC;
- mb->cmdr_off = CMDR_OFF;
- mb->cmdr_size = udev->cmdr_size;
- WARN_ON(!PAGE_ALIGNED(udev->data_off));
- WARN_ON(udev->data_size % PAGE_SIZE);
- WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
- info->version = __stringify(TCMU_MAILBOX_VERSION);
- info->mem[0].name = "tcm-user command & data buffer";
- info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
- info->mem[0].size = TCMU_RING_SIZE;
- info->mem[0].memtype = UIO_MEM_VIRTUAL;
- info->irqcontrol = tcmu_irqcontrol;
- info->irq = UIO_IRQ_CUSTOM;
- info->mmap = tcmu_mmap;
- info->open = tcmu_open;
- info->release = tcmu_release;
- ret = uio_register_device(tcmu_root_device, info);
- if (ret)
- goto err_register;
-
- if (dev->dev_attrib.hw_block_size == 0)
- dev->dev_attrib.hw_block_size = 512;
-
- dev->dev_attrib.hw_max_sectors = 128;
- dev->dev_attrib.hw_queue_depth = 128;
- ret = tcmu_netlink_event(TCMU_CMD_ADDED_DEVICE, udev->uio_info.name,
- udev->uio_info.uio_dev->minor);
- if (ret)
- goto err_netlink;
- return 0;
- err_netlink:
- uio_unregister_device(&udev->uio_info);
- err_register:
- vfree(udev->mb_addr);
- err_vzalloc:
- kfree(info->name);
- return ret;
- }
- static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
- {
- if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
- kmem_cache_free(tcmu_cmd_cache, cmd);
- return 0;
- }
- return -EINVAL;
- }
- static void tcmu_dev_call_rcu(struct rcu_head *p)
- {
- struct se_device *dev = container_of(p, struct se_device, rcu_head);
- struct tcmu_dev *udev = TCMU_DEV(dev);
- kfree(udev);
- }
- static void tcmu_free_device(struct se_device *dev)
- {
- struct tcmu_dev *udev = TCMU_DEV(dev);
- struct tcmu_cmd *cmd;
- bool all_expired = true;
- int i;
- del_timer_sync(&udev->timeout);
- vfree(udev->mb_addr);
-
- spin_lock_irq(&udev->commands_lock);
- idr_for_each_entry(&udev->commands, cmd, i) {
- if (tcmu_check_and_free_pending_cmd(cmd) != 0)
- all_expired = false;
- }
- idr_destroy(&udev->commands);
- spin_unlock_irq(&udev->commands_lock);
- WARN_ON(!all_expired);
-
- if (udev->uio_info.uio_dev) {
- tcmu_netlink_event(TCMU_CMD_REMOVED_DEVICE, udev->uio_info.name,
- udev->uio_info.uio_dev->minor);
- uio_unregister_device(&udev->uio_info);
- kfree(udev->uio_info.name);
- kfree(udev->name);
- }
- call_rcu(&dev->rcu_head, tcmu_dev_call_rcu);
- }
- enum {
- Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_err,
- };
- static match_table_t tokens = {
- {Opt_dev_config, "dev_config=%s"},
- {Opt_dev_size, "dev_size=%u"},
- {Opt_hw_block_size, "hw_block_size=%u"},
- {Opt_err, NULL}
- };
- static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev,
- const char *page, ssize_t count)
- {
- struct tcmu_dev *udev = TCMU_DEV(dev);
- char *orig, *ptr, *opts, *arg_p;
- substring_t args[MAX_OPT_ARGS];
- int ret = 0, token;
- unsigned long tmp_ul;
- opts = kstrdup(page, GFP_KERNEL);
- if (!opts)
- return -ENOMEM;
- orig = opts;
- while ((ptr = strsep(&opts, ",\n")) != NULL) {
- if (!*ptr)
- continue;
- token = match_token(ptr, tokens, args);
- switch (token) {
- case Opt_dev_config:
- if (match_strlcpy(udev->dev_config, &args[0],
- TCMU_CONFIG_LEN) == 0) {
- ret = -EINVAL;
- break;
- }
- pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);
- break;
- case Opt_dev_size:
- arg_p = match_strdup(&args[0]);
- if (!arg_p) {
- ret = -ENOMEM;
- break;
- }
- ret = kstrtoul(arg_p, 0, (unsigned long *) &udev->dev_size);
- kfree(arg_p);
- if (ret < 0)
- pr_err("kstrtoul() failed for dev_size=\n");
- break;
- case Opt_hw_block_size:
- arg_p = match_strdup(&args[0]);
- if (!arg_p) {
- ret = -ENOMEM;
- break;
- }
- ret = kstrtoul(arg_p, 0, &tmp_ul);
- kfree(arg_p);
- if (ret < 0) {
- pr_err("kstrtoul() failed for hw_block_size=\n");
- break;
- }
- if (!tmp_ul) {
- pr_err("hw_block_size must be nonzero\n");
- break;
- }
- dev->dev_attrib.hw_block_size = tmp_ul;
- break;
- default:
- break;
- }
- }
- kfree(orig);
- return (!ret) ? count : ret;
- }
- static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b)
- {
- struct tcmu_dev *udev = TCMU_DEV(dev);
- ssize_t bl = 0;
- bl = sprintf(b + bl, "Config: %s ",
- udev->dev_config[0] ? udev->dev_config : "NULL");
- bl += sprintf(b + bl, "Size: %zu\n", udev->dev_size);
- return bl;
- }
- static sector_t tcmu_get_blocks(struct se_device *dev)
- {
- struct tcmu_dev *udev = TCMU_DEV(dev);
- return div_u64(udev->dev_size - dev->dev_attrib.block_size,
- dev->dev_attrib.block_size);
- }
- static sense_reason_t
- tcmu_parse_cdb(struct se_cmd *cmd)
- {
- return passthrough_parse_cdb(cmd, tcmu_queue_cmd);
- }
- static const struct target_backend_ops tcmu_ops = {
- .name = "user",
- .owner = THIS_MODULE,
- .transport_flags = TRANSPORT_FLAG_PASSTHROUGH,
- .attach_hba = tcmu_attach_hba,
- .detach_hba = tcmu_detach_hba,
- .alloc_device = tcmu_alloc_device,
- .configure_device = tcmu_configure_device,
- .free_device = tcmu_free_device,
- .parse_cdb = tcmu_parse_cdb,
- .set_configfs_dev_params = tcmu_set_configfs_dev_params,
- .show_configfs_dev_params = tcmu_show_configfs_dev_params,
- .get_device_type = sbc_get_device_type,
- .get_blocks = tcmu_get_blocks,
- .tb_dev_attrib_attrs = passthrough_attrib_attrs,
- };
- static int __init tcmu_module_init(void)
- {
- int ret;
- BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0);
- tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache",
- sizeof(struct tcmu_cmd),
- __alignof__(struct tcmu_cmd),
- 0, NULL);
- if (!tcmu_cmd_cache)
- return -ENOMEM;
- tcmu_root_device = root_device_register("tcm_user");
- if (IS_ERR(tcmu_root_device)) {
- ret = PTR_ERR(tcmu_root_device);
- goto out_free_cache;
- }
- ret = genl_register_family(&tcmu_genl_family);
- if (ret < 0) {
- goto out_unreg_device;
- }
- ret = transport_backend_register(&tcmu_ops);
- if (ret)
- goto out_unreg_genl;
- return 0;
- out_unreg_genl:
- genl_unregister_family(&tcmu_genl_family);
- out_unreg_device:
- root_device_unregister(tcmu_root_device);
- out_free_cache:
- kmem_cache_destroy(tcmu_cmd_cache);
- return ret;
- }
- static void __exit tcmu_module_exit(void)
- {
- target_backend_unregister(&tcmu_ops);
- genl_unregister_family(&tcmu_genl_family);
- root_device_unregister(tcmu_root_device);
- kmem_cache_destroy(tcmu_cmd_cache);
- }
- MODULE_DESCRIPTION("TCM USER subsystem plugin");
- MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
- MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
- MODULE_LICENSE("GPL");
- module_init(tcmu_module_init);
- module_exit(tcmu_module_exit);
|