123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758 |
- #include <linux/export.h>
- #include <linux/slab.h>
- #include <linux/scatterlist.h>
- #include <linux/highmem.h>
- #include <linux/kmemleak.h>
- struct scatterlist *sg_next(struct scatterlist *sg)
- {
- #ifdef CONFIG_DEBUG_SG
- BUG_ON(sg->sg_magic != SG_MAGIC);
- #endif
- if (sg_is_last(sg))
- return NULL;
- sg++;
- if (unlikely(sg_is_chain(sg)))
- sg = sg_chain_ptr(sg);
- return sg;
- }
- EXPORT_SYMBOL(sg_next);
- int sg_nents(struct scatterlist *sg)
- {
- int nents;
- for (nents = 0; sg; sg = sg_next(sg))
- nents++;
- return nents;
- }
- EXPORT_SYMBOL(sg_nents);
- int sg_nents_for_len(struct scatterlist *sg, u64 len)
- {
- int nents;
- u64 total;
- if (!len)
- return 0;
- for (nents = 0, total = 0; sg; sg = sg_next(sg)) {
- nents++;
- total += sg->length;
- if (total >= len)
- return nents;
- }
- return -EINVAL;
- }
- EXPORT_SYMBOL(sg_nents_for_len);
- struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents)
- {
- struct scatterlist *sg, *ret = NULL;
- unsigned int i;
- for_each_sg(sgl, sg, nents, i)
- ret = sg;
- #ifdef CONFIG_DEBUG_SG
- BUG_ON(sgl[0].sg_magic != SG_MAGIC);
- BUG_ON(!sg_is_last(ret));
- #endif
- return ret;
- }
- EXPORT_SYMBOL(sg_last);
- void sg_init_table(struct scatterlist *sgl, unsigned int nents)
- {
- memset(sgl, 0, sizeof(*sgl) * nents);
- #ifdef CONFIG_DEBUG_SG
- {
- unsigned int i;
- for (i = 0; i < nents; i++)
- sgl[i].sg_magic = SG_MAGIC;
- }
- #endif
- sg_mark_end(&sgl[nents - 1]);
- }
- EXPORT_SYMBOL(sg_init_table);
- void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)
- {
- sg_init_table(sg, 1);
- sg_set_buf(sg, buf, buflen);
- }
- EXPORT_SYMBOL(sg_init_one);
- static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
- {
- if (nents == SG_MAX_SINGLE_ALLOC) {
-
- void *ptr = (void *) __get_free_page(gfp_mask);
- kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
- return ptr;
- } else
- return kmalloc(nents * sizeof(struct scatterlist), gfp_mask);
- }
- static void sg_kfree(struct scatterlist *sg, unsigned int nents)
- {
- if (nents == SG_MAX_SINGLE_ALLOC) {
- kmemleak_free(sg);
- free_page((unsigned long) sg);
- } else
- kfree(sg);
- }
- void __sg_free_table(struct sg_table *table, unsigned int max_ents,
- bool skip_first_chunk, sg_free_fn *free_fn)
- {
- struct scatterlist *sgl, *next;
- if (unlikely(!table->sgl))
- return;
- sgl = table->sgl;
- while (table->orig_nents) {
- unsigned int alloc_size = table->orig_nents;
- unsigned int sg_size;
-
- if (alloc_size > max_ents) {
- next = sg_chain_ptr(&sgl[max_ents - 1]);
- alloc_size = max_ents;
- sg_size = alloc_size - 1;
- } else {
- sg_size = alloc_size;
- next = NULL;
- }
- table->orig_nents -= sg_size;
- if (skip_first_chunk)
- skip_first_chunk = false;
- else
- free_fn(sgl, alloc_size);
- sgl = next;
- }
- table->sgl = NULL;
- }
- EXPORT_SYMBOL(__sg_free_table);
- void sg_free_table(struct sg_table *table)
- {
- __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
- }
- EXPORT_SYMBOL(sg_free_table);
- int __sg_alloc_table(struct sg_table *table, unsigned int nents,
- unsigned int max_ents, struct scatterlist *first_chunk,
- gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
- {
- struct scatterlist *sg, *prv;
- unsigned int left;
- memset(table, 0, sizeof(*table));
- if (nents == 0)
- return -EINVAL;
- #ifndef CONFIG_ARCH_HAS_SG_CHAIN
- if (WARN_ON_ONCE(nents > max_ents))
- return -EINVAL;
- #endif
- left = nents;
- prv = NULL;
- do {
- unsigned int sg_size, alloc_size = left;
- if (alloc_size > max_ents) {
- alloc_size = max_ents;
- sg_size = alloc_size - 1;
- } else
- sg_size = alloc_size;
- left -= sg_size;
- if (first_chunk) {
- sg = first_chunk;
- first_chunk = NULL;
- } else {
- sg = alloc_fn(alloc_size, gfp_mask);
- }
- if (unlikely(!sg)) {
-
- if (prv)
- table->nents = ++table->orig_nents;
- return -ENOMEM;
- }
- sg_init_table(sg, alloc_size);
- table->nents = table->orig_nents += sg_size;
-
- if (prv)
- sg_chain(prv, max_ents, sg);
- else
- table->sgl = sg;
-
- if (!left)
- sg_mark_end(&sg[sg_size - 1]);
- prv = sg;
- } while (left);
- return 0;
- }
- EXPORT_SYMBOL(__sg_alloc_table);
- int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
- {
- int ret;
- ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
- NULL, gfp_mask, sg_kmalloc);
- if (unlikely(ret))
- __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
- return ret;
- }
- EXPORT_SYMBOL(sg_alloc_table);
- int sg_alloc_table_from_pages(struct sg_table *sgt,
- struct page **pages, unsigned int n_pages,
- unsigned long offset, unsigned long size,
- gfp_t gfp_mask)
- {
- unsigned int chunks;
- unsigned int i;
- unsigned int cur_page;
- int ret;
- struct scatterlist *s;
-
- chunks = 1;
- for (i = 1; i < n_pages; ++i)
- if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1)
- ++chunks;
- ret = sg_alloc_table(sgt, chunks, gfp_mask);
- if (unlikely(ret))
- return ret;
-
- cur_page = 0;
- for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
- unsigned long chunk_size;
- unsigned int j;
-
- for (j = cur_page + 1; j < n_pages; ++j)
- if (page_to_pfn(pages[j]) !=
- page_to_pfn(pages[j - 1]) + 1)
- break;
- chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
- sg_set_page(s, pages[cur_page], min(size, chunk_size), offset);
- size -= chunk_size;
- offset = 0;
- cur_page = j;
- }
- return 0;
- }
- EXPORT_SYMBOL(sg_alloc_table_from_pages);
- void __sg_page_iter_start(struct sg_page_iter *piter,
- struct scatterlist *sglist, unsigned int nents,
- unsigned long pgoffset)
- {
- piter->__pg_advance = 0;
- piter->__nents = nents;
- piter->sg = sglist;
- piter->sg_pgoffset = pgoffset;
- }
- EXPORT_SYMBOL(__sg_page_iter_start);
- static int sg_page_count(struct scatterlist *sg)
- {
- return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
- }
- bool __sg_page_iter_next(struct sg_page_iter *piter)
- {
- if (!piter->__nents || !piter->sg)
- return false;
- piter->sg_pgoffset += piter->__pg_advance;
- piter->__pg_advance = 1;
- while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
- piter->sg_pgoffset -= sg_page_count(piter->sg);
- piter->sg = sg_next(piter->sg);
- if (!--piter->__nents || !piter->sg)
- return false;
- }
- return true;
- }
- EXPORT_SYMBOL(__sg_page_iter_next);
- void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
- unsigned int nents, unsigned int flags)
- {
- memset(miter, 0, sizeof(struct sg_mapping_iter));
- __sg_page_iter_start(&miter->piter, sgl, nents, 0);
- WARN_ON(!(flags & (SG_MITER_TO_SG | SG_MITER_FROM_SG)));
- miter->__flags = flags;
- }
- EXPORT_SYMBOL(sg_miter_start);
- static bool sg_miter_get_next_page(struct sg_mapping_iter *miter)
- {
- if (!miter->__remaining) {
- struct scatterlist *sg;
- if (!__sg_page_iter_next(&miter->piter))
- return false;
- sg = miter->piter.sg;
- miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset;
- miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT;
- miter->__offset &= PAGE_SIZE - 1;
- miter->__remaining = sg->offset + sg->length -
- (miter->piter.sg_pgoffset << PAGE_SHIFT) -
- miter->__offset;
- miter->__remaining = min_t(unsigned long, miter->__remaining,
- PAGE_SIZE - miter->__offset);
- }
- return true;
- }
- bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset)
- {
- sg_miter_stop(miter);
- while (offset) {
- off_t consumed;
- if (!sg_miter_get_next_page(miter))
- return false;
- consumed = min_t(off_t, offset, miter->__remaining);
- miter->__offset += consumed;
- miter->__remaining -= consumed;
- offset -= consumed;
- }
- return true;
- }
- EXPORT_SYMBOL(sg_miter_skip);
- bool sg_miter_next(struct sg_mapping_iter *miter)
- {
- sg_miter_stop(miter);
-
- if (!sg_miter_get_next_page(miter))
- return false;
- miter->page = sg_page_iter_page(&miter->piter);
- miter->consumed = miter->length = miter->__remaining;
- if (miter->__flags & SG_MITER_ATOMIC)
- miter->addr = kmap_atomic(miter->page) + miter->__offset;
- else
- miter->addr = kmap(miter->page) + miter->__offset;
- return true;
- }
- EXPORT_SYMBOL(sg_miter_next);
- void sg_miter_stop(struct sg_mapping_iter *miter)
- {
- WARN_ON(miter->consumed > miter->length);
-
- if (miter->addr) {
- miter->__offset += miter->consumed;
- miter->__remaining -= miter->consumed;
- if ((miter->__flags & SG_MITER_TO_SG) &&
- !PageSlab(miter->page))
- flush_kernel_dcache_page(miter->page);
- if (miter->__flags & SG_MITER_ATOMIC) {
- WARN_ON_ONCE(preemptible());
- kunmap_atomic(miter->addr);
- } else
- kunmap(miter->page);
- miter->page = NULL;
- miter->addr = NULL;
- miter->length = 0;
- miter->consumed = 0;
- }
- }
- EXPORT_SYMBOL(sg_miter_stop);
- size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
- size_t buflen, off_t skip, bool to_buffer)
- {
- unsigned int offset = 0;
- struct sg_mapping_iter miter;
- unsigned long flags;
- unsigned int sg_flags = SG_MITER_ATOMIC;
- if (to_buffer)
- sg_flags |= SG_MITER_FROM_SG;
- else
- sg_flags |= SG_MITER_TO_SG;
- sg_miter_start(&miter, sgl, nents, sg_flags);
- if (!sg_miter_skip(&miter, skip))
- return false;
- local_irq_save(flags);
- while (sg_miter_next(&miter) && offset < buflen) {
- unsigned int len;
- len = min(miter.length, buflen - offset);
- if (to_buffer)
- memcpy(buf + offset, miter.addr, len);
- else
- memcpy(miter.addr, buf + offset, len);
- offset += len;
- }
- sg_miter_stop(&miter);
- local_irq_restore(flags);
- return offset;
- }
- EXPORT_SYMBOL(sg_copy_buffer);
- size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
- const void *buf, size_t buflen)
- {
- return sg_copy_buffer(sgl, nents, (void *)buf, buflen, 0, false);
- }
- EXPORT_SYMBOL(sg_copy_from_buffer);
- size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
- void *buf, size_t buflen)
- {
- return sg_copy_buffer(sgl, nents, buf, buflen, 0, true);
- }
- EXPORT_SYMBOL(sg_copy_to_buffer);
- size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
- const void *buf, size_t buflen, off_t skip)
- {
- return sg_copy_buffer(sgl, nents, (void *)buf, buflen, skip, false);
- }
- EXPORT_SYMBOL(sg_pcopy_from_buffer);
- size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
- void *buf, size_t buflen, off_t skip)
- {
- return sg_copy_buffer(sgl, nents, buf, buflen, skip, true);
- }
- EXPORT_SYMBOL(sg_pcopy_to_buffer);
|