123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778 |
- #include <linux/kernel.h>
- #include <linux/export.h>
- #include <linux/mm.h>
- #include <linux/kdev_t.h>
- #include <linux/gfp.h>
- #include <linux/bio.h>
- #include <linux/fs.h>
- #include <linux/buffer_head.h>
- #include <linux/blkdev.h>
- #include <linux/highmem.h>
- #include <linux/prefetch.h>
- #include <linux/mpage.h>
- #include <linux/mm_inline.h>
- #include <linux/writeback.h>
- #include <linux/backing-dev.h>
- #include <linux/pagevec.h>
- #include <linux/cleancache.h>
- #include "internal.h"
- #define CREATE_TRACE_POINTS
- #include <trace/events/android_fs.h>
- EXPORT_TRACEPOINT_SYMBOL(android_fs_datawrite_start);
- EXPORT_TRACEPOINT_SYMBOL(android_fs_datawrite_end);
- EXPORT_TRACEPOINT_SYMBOL(android_fs_dataread_start);
- EXPORT_TRACEPOINT_SYMBOL(android_fs_dataread_end);
- static void mpage_end_io(struct bio *bio)
- {
- struct bio_vec *bv;
- int i;
- if (trace_android_fs_dataread_end_enabled() &&
- (bio_data_dir(bio) == READ)) {
- struct page *first_page = bio->bi_io_vec[0].bv_page;
- if (first_page != NULL)
- trace_android_fs_dataread_end(first_page->mapping->host,
- page_offset(first_page),
- bio->bi_iter.bi_size);
- }
- bio_for_each_segment_all(bv, bio, i) {
- struct page *page = bv->bv_page;
- page_endio(page, op_is_write(bio_op(bio)), bio->bi_error);
- }
- bio_put(bio);
- }
- static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio)
- {
- if (trace_android_fs_dataread_start_enabled() && (op == REQ_OP_READ)) {
- struct page *first_page = bio->bi_io_vec[0].bv_page;
- if (first_page != NULL) {
- char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
- path = android_fstrace_get_pathname(pathbuf,
- MAX_TRACE_PATHBUF_LEN,
- first_page->mapping->host);
- trace_android_fs_dataread_start(
- first_page->mapping->host,
- page_offset(first_page),
- bio->bi_iter.bi_size,
- current->pid,
- path,
- current->comm);
- }
- }
- bio->bi_end_io = mpage_end_io;
- bio_set_op_attrs(bio, op, op_flags);
- guard_bio_eod(op, bio);
- submit_bio(bio);
- return NULL;
- }
- static struct bio *
- mpage_alloc(struct block_device *bdev,
- sector_t first_sector, int nr_vecs,
- gfp_t gfp_flags)
- {
- struct bio *bio;
-
- gfp_flags &= GFP_KERNEL;
- bio = bio_alloc(gfp_flags, nr_vecs);
- if (bio == NULL && (current->flags & PF_MEMALLOC)) {
- while (!bio && (nr_vecs /= 2))
- bio = bio_alloc(gfp_flags, nr_vecs);
- }
- if (bio) {
- bio->bi_bdev = bdev;
- bio->bi_iter.bi_sector = first_sector;
- }
- return bio;
- }
- static void
- map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
- {
- struct inode *inode = page->mapping->host;
- struct buffer_head *page_bh, *head;
- int block = 0;
- if (!page_has_buffers(page)) {
-
- if (inode->i_blkbits == PAGE_SHIFT &&
- buffer_uptodate(bh)) {
- SetPageUptodate(page);
- return;
- }
- create_empty_buffers(page, i_blocksize(inode), 0);
- }
- head = page_buffers(page);
- page_bh = head;
- do {
- if (block == page_block) {
- page_bh->b_state = bh->b_state;
- page_bh->b_bdev = bh->b_bdev;
- page_bh->b_blocknr = bh->b_blocknr;
- break;
- }
- page_bh = page_bh->b_this_page;
- block++;
- } while (page_bh != head);
- }
- static struct bio *
- do_mpage_readpage(struct bio *bio, struct page *page, unsigned nr_pages,
- sector_t *last_block_in_bio, struct buffer_head *map_bh,
- unsigned long *first_logical_block, get_block_t get_block,
- gfp_t gfp)
- {
- struct inode *inode = page->mapping->host;
- const unsigned blkbits = inode->i_blkbits;
- const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
- const unsigned blocksize = 1 << blkbits;
- sector_t block_in_file;
- sector_t last_block;
- sector_t last_block_in_file;
- sector_t blocks[MAX_BUF_PER_PAGE];
- unsigned page_block;
- unsigned first_hole = blocks_per_page;
- struct block_device *bdev = NULL;
- int length;
- int fully_mapped = 1;
- unsigned nblocks;
- unsigned relative_block;
- if (page_has_buffers(page))
- goto confused;
- block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
- last_block = block_in_file + nr_pages * blocks_per_page;
- last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
- if (last_block > last_block_in_file)
- last_block = last_block_in_file;
- page_block = 0;
-
- nblocks = map_bh->b_size >> blkbits;
- if (buffer_mapped(map_bh) && block_in_file > *first_logical_block &&
- block_in_file < (*first_logical_block + nblocks)) {
- unsigned map_offset = block_in_file - *first_logical_block;
- unsigned last = nblocks - map_offset;
- for (relative_block = 0; ; relative_block++) {
- if (relative_block == last) {
- clear_buffer_mapped(map_bh);
- break;
- }
- if (page_block == blocks_per_page)
- break;
- blocks[page_block] = map_bh->b_blocknr + map_offset +
- relative_block;
- page_block++;
- block_in_file++;
- }
- bdev = map_bh->b_bdev;
- }
-
- map_bh->b_page = page;
- while (page_block < blocks_per_page) {
- map_bh->b_state = 0;
- map_bh->b_size = 0;
- if (block_in_file < last_block) {
- map_bh->b_size = (last_block-block_in_file) << blkbits;
- if (get_block(inode, block_in_file, map_bh, 0))
- goto confused;
- *first_logical_block = block_in_file;
- }
- if (!buffer_mapped(map_bh)) {
- fully_mapped = 0;
- if (first_hole == blocks_per_page)
- first_hole = page_block;
- page_block++;
- block_in_file++;
- continue;
- }
-
- if (buffer_uptodate(map_bh)) {
- map_buffer_to_page(page, map_bh, page_block);
- goto confused;
- }
-
- if (first_hole != blocks_per_page)
- goto confused;
-
- if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
- goto confused;
- nblocks = map_bh->b_size >> blkbits;
- for (relative_block = 0; ; relative_block++) {
- if (relative_block == nblocks) {
- clear_buffer_mapped(map_bh);
- break;
- } else if (page_block == blocks_per_page)
- break;
- blocks[page_block] = map_bh->b_blocknr+relative_block;
- page_block++;
- block_in_file++;
- }
- bdev = map_bh->b_bdev;
- }
- if (first_hole != blocks_per_page) {
- zero_user_segment(page, first_hole << blkbits, PAGE_SIZE);
- if (first_hole == 0) {
- SetPageUptodate(page);
- unlock_page(page);
- goto out;
- }
- } else if (fully_mapped) {
- SetPageMappedToDisk(page);
- }
- if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
- cleancache_get_page(page) == 0) {
- SetPageUptodate(page);
- goto confused;
- }
-
- if (bio && (*last_block_in_bio != blocks[0] - 1))
- bio = mpage_bio_submit(REQ_OP_READ, 0, bio);
- alloc_new:
- if (bio == NULL) {
- if (first_hole == blocks_per_page) {
- if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
- page))
- goto out;
- }
- bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
- min_t(int, nr_pages, BIO_MAX_PAGES), gfp);
- if (bio == NULL)
- goto confused;
- }
- length = first_hole << blkbits;
- if (bio_add_page(bio, page, length, 0) < length) {
- bio = mpage_bio_submit(REQ_OP_READ, 0, bio);
- goto alloc_new;
- }
- relative_block = block_in_file - *first_logical_block;
- nblocks = map_bh->b_size >> blkbits;
- if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
- (first_hole != blocks_per_page))
- bio = mpage_bio_submit(REQ_OP_READ, 0, bio);
- else
- *last_block_in_bio = blocks[blocks_per_page - 1];
- out:
- return bio;
- confused:
- if (bio)
- bio = mpage_bio_submit(REQ_OP_READ, 0, bio);
- if (!PageUptodate(page))
- block_read_full_page(page, get_block);
- else
- unlock_page(page);
- goto out;
- }
- int
- mpage_readpages(struct address_space *mapping, struct list_head *pages,
- unsigned nr_pages, get_block_t get_block)
- {
- struct bio *bio = NULL;
- unsigned page_idx;
- sector_t last_block_in_bio = 0;
- struct buffer_head map_bh;
- unsigned long first_logical_block = 0;
- gfp_t gfp = readahead_gfp_mask(mapping);
- map_bh.b_state = 0;
- map_bh.b_size = 0;
- for (page_idx = 0; page_idx < nr_pages; page_idx++) {
- struct page *page = lru_to_page(pages);
- prefetchw(&page->flags);
- list_del(&page->lru);
- if (!add_to_page_cache_lru(page, mapping,
- page->index,
- gfp)) {
- bio = do_mpage_readpage(bio, page,
- nr_pages - page_idx,
- &last_block_in_bio, &map_bh,
- &first_logical_block,
- get_block, gfp);
- }
- put_page(page);
- }
- BUG_ON(!list_empty(pages));
- if (bio)
- mpage_bio_submit(REQ_OP_READ, 0, bio);
- return 0;
- }
- EXPORT_SYMBOL(mpage_readpages);
- int mpage_readpage(struct page *page, get_block_t get_block)
- {
- struct bio *bio = NULL;
- sector_t last_block_in_bio = 0;
- struct buffer_head map_bh;
- unsigned long first_logical_block = 0;
- gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
- map_bh.b_state = 0;
- map_bh.b_size = 0;
- bio = do_mpage_readpage(bio, page, 1, &last_block_in_bio,
- &map_bh, &first_logical_block, get_block, gfp);
- if (bio)
- mpage_bio_submit(REQ_OP_READ, 0, bio);
- return 0;
- }
- EXPORT_SYMBOL(mpage_readpage);
- struct mpage_data {
- struct bio *bio;
- sector_t last_block_in_bio;
- get_block_t *get_block;
- unsigned use_writepage;
- };
- static void clean_buffers(struct page *page, unsigned first_unmapped)
- {
- unsigned buffer_counter = 0;
- struct buffer_head *bh, *head;
- if (!page_has_buffers(page))
- return;
- head = page_buffers(page);
- bh = head;
- do {
- if (buffer_counter++ == first_unmapped)
- break;
- clear_buffer_dirty(bh);
- bh = bh->b_this_page;
- } while (bh != head);
-
- if (buffer_heads_over_limit && PageUptodate(page))
- try_to_free_buffers(page);
- }
- void clean_page_buffers(struct page *page)
- {
- clean_buffers(page, ~0U);
- }
- static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
- void *data)
- {
- struct mpage_data *mpd = data;
- struct bio *bio = mpd->bio;
- struct address_space *mapping = page->mapping;
- struct inode *inode = page->mapping->host;
- const unsigned blkbits = inode->i_blkbits;
- unsigned long end_index;
- const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
- sector_t last_block;
- sector_t block_in_file;
- sector_t blocks[MAX_BUF_PER_PAGE];
- unsigned page_block;
- unsigned first_unmapped = blocks_per_page;
- struct block_device *bdev = NULL;
- int boundary = 0;
- sector_t boundary_block = 0;
- struct block_device *boundary_bdev = NULL;
- int length;
- struct buffer_head map_bh;
- loff_t i_size = i_size_read(inode);
- int ret = 0;
- int op_flags = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : 0);
- if (page_has_buffers(page)) {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh = head;
-
- page_block = 0;
- do {
- BUG_ON(buffer_locked(bh));
- if (!buffer_mapped(bh)) {
-
- if (buffer_dirty(bh))
- goto confused;
- if (first_unmapped == blocks_per_page)
- first_unmapped = page_block;
- continue;
- }
- if (first_unmapped != blocks_per_page)
- goto confused;
- if (!buffer_dirty(bh) || !buffer_uptodate(bh))
- goto confused;
- if (page_block) {
- if (bh->b_blocknr != blocks[page_block-1] + 1)
- goto confused;
- }
- blocks[page_block++] = bh->b_blocknr;
- boundary = buffer_boundary(bh);
- if (boundary) {
- boundary_block = bh->b_blocknr;
- boundary_bdev = bh->b_bdev;
- }
- bdev = bh->b_bdev;
- } while ((bh = bh->b_this_page) != head);
- if (first_unmapped)
- goto page_is_mapped;
-
- goto confused;
- }
-
- BUG_ON(!PageUptodate(page));
- block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
- last_block = (i_size - 1) >> blkbits;
- map_bh.b_page = page;
- for (page_block = 0; page_block < blocks_per_page; ) {
- map_bh.b_state = 0;
- map_bh.b_size = 1 << blkbits;
- if (mpd->get_block(inode, block_in_file, &map_bh, 1))
- goto confused;
- if (buffer_new(&map_bh))
- unmap_underlying_metadata(map_bh.b_bdev,
- map_bh.b_blocknr);
- if (buffer_boundary(&map_bh)) {
- boundary_block = map_bh.b_blocknr;
- boundary_bdev = map_bh.b_bdev;
- }
- if (page_block) {
- if (map_bh.b_blocknr != blocks[page_block-1] + 1)
- goto confused;
- }
- blocks[page_block++] = map_bh.b_blocknr;
- boundary = buffer_boundary(&map_bh);
- bdev = map_bh.b_bdev;
- if (block_in_file == last_block)
- break;
- block_in_file++;
- }
- BUG_ON(page_block == 0);
- first_unmapped = page_block;
- page_is_mapped:
- end_index = i_size >> PAGE_SHIFT;
- if (page->index >= end_index) {
-
- unsigned offset = i_size & (PAGE_SIZE - 1);
- if (page->index > end_index || !offset)
- goto confused;
- zero_user_segment(page, offset, PAGE_SIZE);
- }
-
- if (bio && mpd->last_block_in_bio != blocks[0] - 1)
- bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
- alloc_new:
- if (bio == NULL) {
- if (first_unmapped == blocks_per_page) {
- if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
- page, wbc))
- goto out;
- }
- bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
- BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH);
- if (bio == NULL)
- goto confused;
- wbc_init_bio(wbc, bio);
- }
-
- wbc_account_io(wbc, page, PAGE_SIZE);
- length = first_unmapped << blkbits;
- if (bio_add_page(bio, page, length, 0) < length) {
- bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
- goto alloc_new;
- }
- clean_buffers(page, first_unmapped);
- BUG_ON(PageWriteback(page));
- set_page_writeback(page);
- unlock_page(page);
- if (boundary || (first_unmapped != blocks_per_page)) {
- bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
- if (boundary_block) {
- write_boundary_block(boundary_bdev,
- boundary_block, 1 << blkbits);
- }
- } else {
- mpd->last_block_in_bio = blocks[blocks_per_page - 1];
- }
- goto out;
- confused:
- if (bio)
- bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
- if (mpd->use_writepage) {
- ret = mapping->a_ops->writepage(page, wbc);
- } else {
- ret = -EAGAIN;
- goto out;
- }
-
- mapping_set_error(mapping, ret);
- out:
- mpd->bio = bio;
- return ret;
- }
- int
- mpage_writepages(struct address_space *mapping,
- struct writeback_control *wbc, get_block_t get_block)
- {
- struct blk_plug plug;
- int ret;
- blk_start_plug(&plug);
- if (!get_block)
- ret = generic_writepages(mapping, wbc);
- else {
- struct mpage_data mpd = {
- .bio = NULL,
- .last_block_in_bio = 0,
- .get_block = get_block,
- .use_writepage = 1,
- };
- ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
- if (mpd.bio) {
- int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
- WRITE_SYNC : 0);
- mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
- }
- }
- blk_finish_plug(&plug);
- return ret;
- }
- EXPORT_SYMBOL(mpage_writepages);
- int mpage_writepage(struct page *page, get_block_t get_block,
- struct writeback_control *wbc)
- {
- struct mpage_data mpd = {
- .bio = NULL,
- .last_block_in_bio = 0,
- .get_block = get_block,
- .use_writepage = 0,
- };
- int ret = __mpage_writepage(page, wbc, &mpd);
- if (mpd.bio) {
- int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
- WRITE_SYNC : 0);
- mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
- }
- return ret;
- }
- EXPORT_SYMBOL(mpage_writepage);
|