1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538 |
- #include <linux/kernel.h>
- #include <linux/syscalls.h>
- #include <linux/fs.h>
- #include <linux/iomap.h>
- #include <linux/mm.h>
- #include <linux/percpu.h>
- #include <linux/slab.h>
- #include <linux/capability.h>
- #include <linux/blkdev.h>
- #include <linux/file.h>
- #include <linux/quotaops.h>
- #include <linux/highmem.h>
- #include <linux/export.h>
- #include <linux/backing-dev.h>
- #include <linux/writeback.h>
- #include <linux/hash.h>
- #include <linux/suspend.h>
- #include <linux/buffer_head.h>
- #include <linux/task_io_accounting_ops.h>
- #include <linux/bio.h>
- #include <linux/notifier.h>
- #include <linux/cpu.h>
- #include <linux/bitops.h>
- #include <linux/mpage.h>
- #include <linux/bit_spinlock.h>
- #include <trace/events/block.h>
- static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
- static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags,
- struct writeback_control *wbc);
- #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
- void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
- {
- bh->b_end_io = handler;
- bh->b_private = private;
- }
- EXPORT_SYMBOL(init_buffer);
- inline void touch_buffer(struct buffer_head *bh)
- {
- trace_block_touch_buffer(bh);
- mark_page_accessed(bh->b_page);
- }
- EXPORT_SYMBOL(touch_buffer);
- void __lock_buffer(struct buffer_head *bh)
- {
- wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
- }
- EXPORT_SYMBOL(__lock_buffer);
- void unlock_buffer(struct buffer_head *bh)
- {
- clear_bit_unlock(BH_Lock, &bh->b_state);
- smp_mb__after_atomic();
- wake_up_bit(&bh->b_state, BH_Lock);
- }
- EXPORT_SYMBOL(unlock_buffer);
- void buffer_check_dirty_writeback(struct page *page,
- bool *dirty, bool *writeback)
- {
- struct buffer_head *head, *bh;
- *dirty = false;
- *writeback = false;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- return;
- if (PageWriteback(page))
- *writeback = true;
- head = page_buffers(page);
- bh = head;
- do {
- if (buffer_locked(bh))
- *writeback = true;
- if (buffer_dirty(bh))
- *dirty = true;
- bh = bh->b_this_page;
- } while (bh != head);
- }
- EXPORT_SYMBOL(buffer_check_dirty_writeback);
- void __wait_on_buffer(struct buffer_head * bh)
- {
- wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
- }
- EXPORT_SYMBOL(__wait_on_buffer);
- static void
- __clear_page_buffers(struct page *page)
- {
- ClearPagePrivate(page);
- set_page_private(page, 0);
- put_page(page);
- }
- static void buffer_io_error(struct buffer_head *bh, char *msg)
- {
- if (!test_bit(BH_Quiet, &bh->b_state))
- printk_ratelimited(KERN_ERR
- "Buffer I/O error on dev %pg, logical block %llu%s\n",
- bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
- }
- static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
- {
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
-
- clear_buffer_uptodate(bh);
- }
- unlock_buffer(bh);
- }
- void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
- {
- __end_buffer_read_notouch(bh, uptodate);
- put_bh(bh);
- }
- EXPORT_SYMBOL(end_buffer_read_sync);
- void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
- {
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- buffer_io_error(bh, ", lost sync page write");
- set_buffer_write_io_error(bh);
- clear_buffer_uptodate(bh);
- }
- unlock_buffer(bh);
- put_bh(bh);
- }
- EXPORT_SYMBOL(end_buffer_write_sync);
- static struct buffer_head *
- __find_get_block_slow(struct block_device *bdev, sector_t block)
- {
- struct inode *bd_inode = bdev->bd_inode;
- struct address_space *bd_mapping = bd_inode->i_mapping;
- struct buffer_head *ret = NULL;
- pgoff_t index;
- struct buffer_head *bh;
- struct buffer_head *head;
- struct page *page;
- int all_mapped = 1;
- static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
- index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
- page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
- if (!page)
- goto out;
- spin_lock(&bd_mapping->private_lock);
- if (!page_has_buffers(page))
- goto out_unlock;
- head = page_buffers(page);
- bh = head;
- do {
- if (!buffer_mapped(bh))
- all_mapped = 0;
- else if (bh->b_blocknr == block) {
- ret = bh;
- get_bh(bh);
- goto out_unlock;
- }
- bh = bh->b_this_page;
- } while (bh != head);
-
- ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
- if (all_mapped && __ratelimit(&last_warned)) {
- printk("__find_get_block_slow() failed. block=%llu, "
- "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
- "device %pg blocksize: %d\n",
- (unsigned long long)block,
- (unsigned long long)bh->b_blocknr,
- bh->b_state, bh->b_size, bdev,
- 1 << bd_inode->i_blkbits);
- }
- out_unlock:
- spin_unlock(&bd_mapping->private_lock);
- put_page(page);
- out:
- return ret;
- }
- static void free_more_memory(void)
- {
- struct zoneref *z;
- int nid;
- wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
- yield();
- for_each_online_node(nid) {
- z = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
- gfp_zone(GFP_NOFS), NULL);
- if (z->zone)
- try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
- GFP_NOFS, NULL);
- }
- }
- static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
- {
- unsigned long flags;
- struct buffer_head *first;
- struct buffer_head *tmp;
- struct page *page;
- int page_uptodate = 1;
- BUG_ON(!buffer_async_read(bh));
- page = bh->b_page;
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- clear_buffer_uptodate(bh);
- buffer_io_error(bh, ", async page read");
- SetPageError(page);
- }
-
- first = page_buffers(page);
- local_irq_save(flags);
- bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
- clear_buffer_async_read(bh);
- unlock_buffer(bh);
- tmp = bh;
- do {
- if (!buffer_uptodate(tmp))
- page_uptodate = 0;
- if (buffer_async_read(tmp)) {
- BUG_ON(!buffer_locked(tmp));
- goto still_busy;
- }
- tmp = tmp->b_this_page;
- } while (tmp != bh);
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
-
- if (page_uptodate && !PageError(page))
- SetPageUptodate(page);
- unlock_page(page);
- return;
- still_busy:
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- return;
- }
- void end_buffer_async_write(struct buffer_head *bh, int uptodate)
- {
- unsigned long flags;
- struct buffer_head *first;
- struct buffer_head *tmp;
- struct page *page;
- BUG_ON(!buffer_async_write(bh));
- page = bh->b_page;
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- buffer_io_error(bh, ", lost async page write");
- mapping_set_error(page->mapping, -EIO);
- set_buffer_write_io_error(bh);
- clear_buffer_uptodate(bh);
- SetPageError(page);
- }
- first = page_buffers(page);
- local_irq_save(flags);
- bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
- clear_buffer_async_write(bh);
- unlock_buffer(bh);
- tmp = bh->b_this_page;
- while (tmp != bh) {
- if (buffer_async_write(tmp)) {
- BUG_ON(!buffer_locked(tmp));
- goto still_busy;
- }
- tmp = tmp->b_this_page;
- }
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- end_page_writeback(page);
- return;
- still_busy:
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- return;
- }
- EXPORT_SYMBOL(end_buffer_async_write);
- static void mark_buffer_async_read(struct buffer_head *bh)
- {
- bh->b_end_io = end_buffer_async_read;
- set_buffer_async_read(bh);
- }
- static void mark_buffer_async_write_endio(struct buffer_head *bh,
- bh_end_io_t *handler)
- {
- bh->b_end_io = handler;
- set_buffer_async_write(bh);
- }
- void mark_buffer_async_write(struct buffer_head *bh)
- {
- mark_buffer_async_write_endio(bh, end_buffer_async_write);
- }
- EXPORT_SYMBOL(mark_buffer_async_write);
- static void __remove_assoc_queue(struct buffer_head *bh)
- {
- list_del_init(&bh->b_assoc_buffers);
- WARN_ON(!bh->b_assoc_map);
- if (buffer_write_io_error(bh))
- set_bit(AS_EIO, &bh->b_assoc_map->flags);
- bh->b_assoc_map = NULL;
- }
- int inode_has_buffers(struct inode *inode)
- {
- return !list_empty(&inode->i_data.private_list);
- }
- static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
- {
- struct buffer_head *bh;
- struct list_head *p;
- int err = 0;
- spin_lock(lock);
- repeat:
- list_for_each_prev(p, list) {
- bh = BH_ENTRY(p);
- if (buffer_locked(bh)) {
- get_bh(bh);
- spin_unlock(lock);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- err = -EIO;
- brelse(bh);
- spin_lock(lock);
- goto repeat;
- }
- }
- spin_unlock(lock);
- return err;
- }
- static void do_thaw_one(struct super_block *sb, void *unused)
- {
- while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
- printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
- }
- static void do_thaw_all(struct work_struct *work)
- {
- iterate_supers(do_thaw_one, NULL);
- kfree(work);
- printk(KERN_WARNING "Emergency Thaw complete\n");
- }
- void emergency_thaw_all(void)
- {
- struct work_struct *work;
- work = kmalloc(sizeof(*work), GFP_ATOMIC);
- if (work) {
- INIT_WORK(work, do_thaw_all);
- schedule_work(work);
- }
- }
- int sync_mapping_buffers(struct address_space *mapping)
- {
- struct address_space *buffer_mapping = mapping->private_data;
- if (buffer_mapping == NULL || list_empty(&mapping->private_list))
- return 0;
- return fsync_buffers_list(&buffer_mapping->private_lock,
- &mapping->private_list);
- }
- EXPORT_SYMBOL(sync_mapping_buffers);
- void write_boundary_block(struct block_device *bdev,
- sector_t bblock, unsigned blocksize)
- {
- struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
- if (bh) {
- if (buffer_dirty(bh))
- ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
- put_bh(bh);
- }
- }
- void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
- {
- struct address_space *mapping = inode->i_mapping;
- struct address_space *buffer_mapping = bh->b_page->mapping;
- mark_buffer_dirty(bh);
- if (!mapping->private_data) {
- mapping->private_data = buffer_mapping;
- } else {
- BUG_ON(mapping->private_data != buffer_mapping);
- }
- if (!bh->b_assoc_map) {
- spin_lock(&buffer_mapping->private_lock);
- list_move_tail(&bh->b_assoc_buffers,
- &mapping->private_list);
- bh->b_assoc_map = mapping;
- spin_unlock(&buffer_mapping->private_lock);
- }
- }
- EXPORT_SYMBOL(mark_buffer_dirty_inode);
- static void __set_page_dirty(struct page *page, struct address_space *mapping,
- int warn)
- {
- unsigned long flags;
- spin_lock_irqsave(&mapping->tree_lock, flags);
- if (page->mapping) {
- WARN_ON_ONCE(warn && !PageUptodate(page));
- account_page_dirtied(page, mapping);
- radix_tree_tag_set(&mapping->page_tree,
- page_index(page), PAGECACHE_TAG_DIRTY);
- }
- spin_unlock_irqrestore(&mapping->tree_lock, flags);
- }
- int __set_page_dirty_buffers(struct page *page)
- {
- int newly_dirty;
- struct address_space *mapping = page_mapping(page);
- if (unlikely(!mapping))
- return !TestSetPageDirty(page);
- spin_lock(&mapping->private_lock);
- if (page_has_buffers(page)) {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh = head;
- do {
- set_buffer_dirty(bh);
- bh = bh->b_this_page;
- } while (bh != head);
- }
-
- lock_page_memcg(page);
- newly_dirty = !TestSetPageDirty(page);
- spin_unlock(&mapping->private_lock);
- if (newly_dirty)
- __set_page_dirty(page, mapping, 1);
- unlock_page_memcg(page);
- if (newly_dirty)
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
- return newly_dirty;
- }
- EXPORT_SYMBOL(__set_page_dirty_buffers);
- static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
- {
- struct buffer_head *bh;
- struct list_head tmp;
- struct address_space *mapping;
- int err = 0, err2;
- struct blk_plug plug;
- INIT_LIST_HEAD(&tmp);
- blk_start_plug(&plug);
- spin_lock(lock);
- while (!list_empty(list)) {
- bh = BH_ENTRY(list->next);
- mapping = bh->b_assoc_map;
- __remove_assoc_queue(bh);
-
- smp_mb();
- if (buffer_dirty(bh) || buffer_locked(bh)) {
- list_add(&bh->b_assoc_buffers, &tmp);
- bh->b_assoc_map = mapping;
- if (buffer_dirty(bh)) {
- get_bh(bh);
- spin_unlock(lock);
-
- write_dirty_buffer(bh, WRITE_SYNC);
-
- brelse(bh);
- spin_lock(lock);
- }
- }
- }
- spin_unlock(lock);
- blk_finish_plug(&plug);
- spin_lock(lock);
- while (!list_empty(&tmp)) {
- bh = BH_ENTRY(tmp.prev);
- get_bh(bh);
- mapping = bh->b_assoc_map;
- __remove_assoc_queue(bh);
-
- smp_mb();
- if (buffer_dirty(bh)) {
- list_add(&bh->b_assoc_buffers,
- &mapping->private_list);
- bh->b_assoc_map = mapping;
- }
- spin_unlock(lock);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- err = -EIO;
- brelse(bh);
- spin_lock(lock);
- }
-
- spin_unlock(lock);
- err2 = osync_buffers_list(lock, list);
- if (err)
- return err;
- else
- return err2;
- }
- void invalidate_inode_buffers(struct inode *inode)
- {
- if (inode_has_buffers(inode)) {
- struct address_space *mapping = &inode->i_data;
- struct list_head *list = &mapping->private_list;
- struct address_space *buffer_mapping = mapping->private_data;
- spin_lock(&buffer_mapping->private_lock);
- while (!list_empty(list))
- __remove_assoc_queue(BH_ENTRY(list->next));
- spin_unlock(&buffer_mapping->private_lock);
- }
- }
- EXPORT_SYMBOL(invalidate_inode_buffers);
- int remove_inode_buffers(struct inode *inode)
- {
- int ret = 1;
- if (inode_has_buffers(inode)) {
- struct address_space *mapping = &inode->i_data;
- struct list_head *list = &mapping->private_list;
- struct address_space *buffer_mapping = mapping->private_data;
- spin_lock(&buffer_mapping->private_lock);
- while (!list_empty(list)) {
- struct buffer_head *bh = BH_ENTRY(list->next);
- if (buffer_dirty(bh)) {
- ret = 0;
- break;
- }
- __remove_assoc_queue(bh);
- }
- spin_unlock(&buffer_mapping->private_lock);
- }
- return ret;
- }
- struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
- int retry)
- {
- struct buffer_head *bh, *head;
- long offset;
- try_again:
- head = NULL;
- offset = PAGE_SIZE;
- while ((offset -= size) >= 0) {
- bh = alloc_buffer_head(GFP_NOFS);
- if (!bh)
- goto no_grow;
- bh->b_this_page = head;
- bh->b_blocknr = -1;
- head = bh;
- bh->b_size = size;
-
- set_bh_page(bh, page, offset);
- }
- return head;
- no_grow:
- if (head) {
- do {
- bh = head;
- head = head->b_this_page;
- free_buffer_head(bh);
- } while (head);
- }
-
- if (!retry)
- return NULL;
-
- free_more_memory();
- goto try_again;
- }
- EXPORT_SYMBOL_GPL(alloc_page_buffers);
- static inline void
- link_dev_buffers(struct page *page, struct buffer_head *head)
- {
- struct buffer_head *bh, *tail;
- bh = head;
- do {
- tail = bh;
- bh = bh->b_this_page;
- } while (bh);
- tail->b_this_page = head;
- attach_page_buffers(page, head);
- }
- static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
- {
- sector_t retval = ~((sector_t)0);
- loff_t sz = i_size_read(bdev->bd_inode);
- if (sz) {
- unsigned int sizebits = blksize_bits(size);
- retval = (sz >> sizebits);
- }
- return retval;
- }
-
- static sector_t
- init_page_buffers(struct page *page, struct block_device *bdev,
- sector_t block, int size)
- {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh = head;
- int uptodate = PageUptodate(page);
- sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
- do {
- if (!buffer_mapped(bh)) {
- init_buffer(bh, NULL, NULL);
- bh->b_bdev = bdev;
- bh->b_blocknr = block;
- if (uptodate)
- set_buffer_uptodate(bh);
- if (block < end_block)
- set_buffer_mapped(bh);
- }
- block++;
- bh = bh->b_this_page;
- } while (bh != head);
-
- return end_block;
- }
- static int
- grow_dev_page(struct block_device *bdev, sector_t block,
- pgoff_t index, int size, int sizebits, gfp_t gfp)
- {
- struct inode *inode = bdev->bd_inode;
- struct page *page;
- struct buffer_head *bh;
- sector_t end_block;
- int ret = 0;
- gfp_t gfp_mask;
- gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
-
- gfp_mask |= __GFP_NOFAIL;
- page = find_or_create_page(inode->i_mapping, index, gfp_mask);
- if (!page)
- return ret;
- BUG_ON(!PageLocked(page));
- if (page_has_buffers(page)) {
- bh = page_buffers(page);
- if (bh->b_size == size) {
- end_block = init_page_buffers(page, bdev,
- (sector_t)index << sizebits,
- size);
- goto done;
- }
- if (!try_to_free_buffers(page))
- goto failed;
- }
-
- bh = alloc_page_buffers(page, size, 0);
- if (!bh)
- goto failed;
-
- spin_lock(&inode->i_mapping->private_lock);
- link_dev_buffers(page, bh);
- end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
- size);
- spin_unlock(&inode->i_mapping->private_lock);
- done:
- ret = (block < end_block) ? 1 : -ENXIO;
- failed:
- unlock_page(page);
- put_page(page);
- return ret;
- }
- static int
- grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
- {
- pgoff_t index;
- int sizebits;
- sizebits = -1;
- do {
- sizebits++;
- } while ((size << sizebits) < PAGE_SIZE);
- index = block >> sizebits;
-
- if (unlikely(index != block >> sizebits)) {
- printk(KERN_ERR "%s: requested out-of-range block %llu for "
- "device %pg\n",
- __func__, (unsigned long long)block,
- bdev);
- return -EIO;
- }
-
- return grow_dev_page(bdev, block, index, size, sizebits, gfp);
- }
- static struct buffer_head *
- __getblk_slow(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
-
- if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
- (size < 512 || size > PAGE_SIZE))) {
- printk(KERN_ERR "getblk(): invalid block size %d requested\n",
- size);
- printk(KERN_ERR "logical block size: %d\n",
- bdev_logical_block_size(bdev));
- dump_stack();
- return NULL;
- }
- for (;;) {
- struct buffer_head *bh;
- int ret;
- bh = __find_get_block(bdev, block, size);
- if (bh)
- return bh;
- ret = grow_buffers(bdev, block, size, gfp);
- if (ret < 0)
- return NULL;
- if (ret == 0)
- free_more_memory();
- }
- }
- void mark_buffer_dirty(struct buffer_head *bh)
- {
- WARN_ON_ONCE(!buffer_uptodate(bh));
- trace_block_dirty_buffer(bh);
-
- if (buffer_dirty(bh)) {
- smp_mb();
- if (buffer_dirty(bh))
- return;
- }
- if (!test_set_buffer_dirty(bh)) {
- struct page *page = bh->b_page;
- struct address_space *mapping = NULL;
- lock_page_memcg(page);
- if (!TestSetPageDirty(page)) {
- mapping = page_mapping(page);
- if (mapping)
- __set_page_dirty(page, mapping, 0);
- }
- unlock_page_memcg(page);
- if (mapping)
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
- }
- }
- EXPORT_SYMBOL(mark_buffer_dirty);
- void __brelse(struct buffer_head * buf)
- {
- if (atomic_read(&buf->b_count)) {
- put_bh(buf);
- return;
- }
- WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
- }
- EXPORT_SYMBOL(__brelse);
- void __bforget(struct buffer_head *bh)
- {
- clear_buffer_dirty(bh);
- if (bh->b_assoc_map) {
- struct address_space *buffer_mapping = bh->b_page->mapping;
- spin_lock(&buffer_mapping->private_lock);
- list_del_init(&bh->b_assoc_buffers);
- bh->b_assoc_map = NULL;
- spin_unlock(&buffer_mapping->private_lock);
- }
- __brelse(bh);
- }
- EXPORT_SYMBOL(__bforget);
- static struct buffer_head *__bread_slow(struct buffer_head *bh)
- {
- lock_buffer(bh);
- if (buffer_uptodate(bh)) {
- unlock_buffer(bh);
- return bh;
- } else {
- get_bh(bh);
- bh->b_end_io = end_buffer_read_sync;
- submit_bh(REQ_OP_READ, 0, bh);
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
- return bh;
- }
- brelse(bh);
- return NULL;
- }
- #define BH_LRU_SIZE 16
- struct bh_lru {
- struct buffer_head *bhs[BH_LRU_SIZE];
- };
- static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
- #ifdef CONFIG_SMP
- #define bh_lru_lock() local_irq_disable()
- #define bh_lru_unlock() local_irq_enable()
- #else
- #define bh_lru_lock() preempt_disable()
- #define bh_lru_unlock() preempt_enable()
- #endif
- static inline void check_irqs_on(void)
- {
- #ifdef irqs_disabled
- BUG_ON(irqs_disabled());
- #endif
- }
- static void bh_lru_install(struct buffer_head *bh)
- {
- struct buffer_head *evictee = NULL;
- check_irqs_on();
- bh_lru_lock();
- if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
- struct buffer_head *bhs[BH_LRU_SIZE];
- int in;
- int out = 0;
- get_bh(bh);
- bhs[out++] = bh;
- for (in = 0; in < BH_LRU_SIZE; in++) {
- struct buffer_head *bh2 =
- __this_cpu_read(bh_lrus.bhs[in]);
- if (bh2 == bh) {
- __brelse(bh2);
- } else {
- if (out >= BH_LRU_SIZE) {
- BUG_ON(evictee != NULL);
- evictee = bh2;
- } else {
- bhs[out++] = bh2;
- }
- }
- }
- while (out < BH_LRU_SIZE)
- bhs[out++] = NULL;
- memcpy(this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
- }
- bh_lru_unlock();
- if (evictee)
- __brelse(evictee);
- }
- static struct buffer_head *
- lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *ret = NULL;
- unsigned int i;
- check_irqs_on();
- bh_lru_lock();
- for (i = 0; i < BH_LRU_SIZE; i++) {
- struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
- if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
- bh->b_size == size) {
- if (i) {
- while (i) {
- __this_cpu_write(bh_lrus.bhs[i],
- __this_cpu_read(bh_lrus.bhs[i - 1]));
- i--;
- }
- __this_cpu_write(bh_lrus.bhs[0], bh);
- }
- get_bh(bh);
- ret = bh;
- break;
- }
- }
- bh_lru_unlock();
- return ret;
- }
- struct buffer_head *
- __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
- if (bh == NULL) {
-
- bh = __find_get_block_slow(bdev, block);
- if (bh)
- bh_lru_install(bh);
- } else
- touch_buffer(bh);
- return bh;
- }
- EXPORT_SYMBOL(__find_get_block);
- struct buffer_head *
- __getblk_gfp(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
- struct buffer_head *bh = __find_get_block(bdev, block, size);
- might_sleep();
- if (bh == NULL)
- bh = __getblk_slow(bdev, block, size, gfp);
- return bh;
- }
- EXPORT_SYMBOL(__getblk_gfp);
- void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *bh = __getblk(bdev, block, size);
- if (likely(bh)) {
- ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
- brelse(bh);
- }
- }
- EXPORT_SYMBOL(__breadahead);
- struct buffer_head *
- __bread_gfp(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
- struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
- if (likely(bh) && !buffer_uptodate(bh))
- bh = __bread_slow(bh);
- return bh;
- }
- EXPORT_SYMBOL(__bread_gfp);
- static void invalidate_bh_lru(void *arg)
- {
- struct bh_lru *b = &get_cpu_var(bh_lrus);
- int i;
- for (i = 0; i < BH_LRU_SIZE; i++) {
- brelse(b->bhs[i]);
- b->bhs[i] = NULL;
- }
- put_cpu_var(bh_lrus);
- }
- static bool has_bh_in_lru(int cpu, void *dummy)
- {
- struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
- int i;
-
- for (i = 0; i < BH_LRU_SIZE; i++) {
- if (b->bhs[i])
- return 1;
- }
- return 0;
- }
- static void __evict_bh_lru(void *arg)
- {
- struct bh_lru *b = &get_cpu_var(bh_lrus);
- struct buffer_head *bh = arg;
- int i;
- for (i = 0; i < BH_LRU_SIZE; i++) {
- if (b->bhs[i] == bh) {
- brelse(b->bhs[i]);
- b->bhs[i] = NULL;
- goto out;
- }
- }
- out:
- put_cpu_var(bh_lrus);
- }
- static bool bh_exists_in_lru(int cpu, void *arg)
- {
- struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
- struct buffer_head *bh = arg;
- int i;
- for (i = 0; i < BH_LRU_SIZE; i++) {
- if (b->bhs[i] == bh)
- return 1;
- }
- return 0;
- }
- void invalidate_bh_lrus(void)
- {
- on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
- }
- EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
- static void evict_bh_lrus(struct buffer_head *bh)
- {
- on_each_cpu_cond(bh_exists_in_lru, __evict_bh_lru, bh, 1, GFP_ATOMIC);
- }
- void set_bh_page(struct buffer_head *bh,
- struct page *page, unsigned long offset)
- {
- bh->b_page = page;
- BUG_ON(offset >= PAGE_SIZE);
- if (PageHighMem(page))
-
- bh->b_data = (char *)(0 + offset);
- else
- bh->b_data = page_address(page) + offset;
- }
- EXPORT_SYMBOL(set_bh_page);
- #define BUFFER_FLAGS_DISCARD \
- (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
- 1 << BH_Delay | 1 << BH_Unwritten)
- static void discard_buffer(struct buffer_head * bh)
- {
- unsigned long b_state, b_state_old;
- lock_buffer(bh);
- clear_buffer_dirty(bh);
- bh->b_bdev = NULL;
- b_state = bh->b_state;
- for (;;) {
- b_state_old = cmpxchg(&bh->b_state, b_state,
- (b_state & ~BUFFER_FLAGS_DISCARD));
- if (b_state_old == b_state)
- break;
- b_state = b_state_old;
- }
- unlock_buffer(bh);
- }
- void block_invalidatepage(struct page *page, unsigned int offset,
- unsigned int length)
- {
- struct buffer_head *head, *bh, *next;
- unsigned int curr_off = 0;
- unsigned int stop = length + offset;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- goto out;
-
- BUG_ON(stop > PAGE_SIZE || stop < length);
- head = page_buffers(page);
- bh = head;
- do {
- unsigned int next_off = curr_off + bh->b_size;
- next = bh->b_this_page;
-
- if (next_off > stop)
- goto out;
-
- if (offset <= curr_off)
- discard_buffer(bh);
- curr_off = next_off;
- bh = next;
- } while (bh != head);
-
- if (offset == 0)
- try_to_release_page(page, 0);
- out:
- return;
- }
- EXPORT_SYMBOL(block_invalidatepage);
- void create_empty_buffers(struct page *page,
- unsigned long blocksize, unsigned long b_state)
- {
- struct buffer_head *bh, *head, *tail;
- head = alloc_page_buffers(page, blocksize, 1);
- bh = head;
- do {
- bh->b_state |= b_state;
- tail = bh;
- bh = bh->b_this_page;
- } while (bh);
- tail->b_this_page = head;
- spin_lock(&page->mapping->private_lock);
- if (PageUptodate(page) || PageDirty(page)) {
- bh = head;
- do {
- if (PageDirty(page))
- set_buffer_dirty(bh);
- if (PageUptodate(page))
- set_buffer_uptodate(bh);
- bh = bh->b_this_page;
- } while (bh != head);
- }
- attach_page_buffers(page, head);
- spin_unlock(&page->mapping->private_lock);
- }
- EXPORT_SYMBOL(create_empty_buffers);
- void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
- {
- struct buffer_head *old_bh;
- might_sleep();
- old_bh = __find_get_block_slow(bdev, block);
- if (old_bh) {
- clear_buffer_dirty(old_bh);
- wait_on_buffer(old_bh);
- clear_buffer_req(old_bh);
- __brelse(old_bh);
- }
- }
- EXPORT_SYMBOL(unmap_underlying_metadata);
- static inline int block_size_bits(unsigned int blocksize)
- {
- return ilog2(blocksize);
- }
- static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
- {
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
- return page_buffers(page);
- }
- int __block_write_full_page(struct inode *inode, struct page *page,
- get_block_t *get_block, struct writeback_control *wbc,
- bh_end_io_t *handler)
- {
- int err;
- sector_t block;
- sector_t last_block;
- struct buffer_head *bh, *head;
- unsigned int blocksize, bbits;
- int nr_underway = 0;
- int write_flags = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : 0);
- head = create_page_buffers(page, inode,
- (1 << BH_Dirty)|(1 << BH_Uptodate));
-
- bh = head;
- blocksize = bh->b_size;
- bbits = block_size_bits(blocksize);
- block = (sector_t)page->index << (PAGE_SHIFT - bbits);
- last_block = (i_size_read(inode) - 1) >> bbits;
-
- do {
- if (block > last_block) {
-
-
- clear_buffer_dirty(bh);
- set_buffer_uptodate(bh);
- } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
- buffer_dirty(bh)) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, block, bh, 1);
- if (err)
- goto recover;
- clear_buffer_delay(bh);
- if (buffer_new(bh)) {
-
- clear_buffer_new(bh);
- unmap_underlying_metadata(bh->b_bdev,
- bh->b_blocknr);
- }
- }
- bh = bh->b_this_page;
- block++;
- } while (bh != head);
- do {
- if (!buffer_mapped(bh))
- continue;
-
- if (wbc->sync_mode != WB_SYNC_NONE) {
- lock_buffer(bh);
- } else if (!trylock_buffer(bh)) {
- redirty_page_for_writepage(wbc, page);
- continue;
- }
- if (test_clear_buffer_dirty(bh)) {
- mark_buffer_async_write_endio(bh, handler);
- } else {
- unlock_buffer(bh);
- }
- } while ((bh = bh->b_this_page) != head);
-
- BUG_ON(PageWriteback(page));
- set_page_writeback(page);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (buffer_async_write(bh)) {
- submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
- nr_underway++;
- }
- bh = next;
- } while (bh != head);
- unlock_page(page);
- err = 0;
- done:
- if (nr_underway == 0) {
-
- end_page_writeback(page);
-
- }
- return err;
- recover:
-
- bh = head;
-
- do {
- if (buffer_mapped(bh) && buffer_dirty(bh) &&
- !buffer_delay(bh)) {
- lock_buffer(bh);
- mark_buffer_async_write_endio(bh, handler);
- } else {
-
- clear_buffer_dirty(bh);
- }
- } while ((bh = bh->b_this_page) != head);
- SetPageError(page);
- BUG_ON(PageWriteback(page));
- mapping_set_error(page->mapping, err);
- set_page_writeback(page);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (buffer_async_write(bh)) {
- clear_buffer_dirty(bh);
- submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
- nr_underway++;
- }
- bh = next;
- } while (bh != head);
- unlock_page(page);
- goto done;
- }
- EXPORT_SYMBOL(__block_write_full_page);
- void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
- {
- unsigned int block_start, block_end;
- struct buffer_head *head, *bh;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- return;
- bh = head = page_buffers(page);
- block_start = 0;
- do {
- block_end = block_start + bh->b_size;
- if (buffer_new(bh)) {
- if (block_end > from && block_start < to) {
- if (!PageUptodate(page)) {
- unsigned start, size;
- start = max(from, block_start);
- size = min(to, block_end) - start;
- zero_user(page, start, size);
- set_buffer_uptodate(bh);
- }
- clear_buffer_new(bh);
- mark_buffer_dirty(bh);
- }
- }
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
- }
- EXPORT_SYMBOL(page_zero_new_buffers);
- static void
- iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
- struct iomap *iomap)
- {
- loff_t offset = block << inode->i_blkbits;
- bh->b_bdev = iomap->bdev;
-
- BUG_ON(offset >= iomap->offset + iomap->length);
- switch (iomap->type) {
- case IOMAP_HOLE:
-
- if (!buffer_uptodate(bh) ||
- (offset >= i_size_read(inode)))
- set_buffer_new(bh);
- break;
- case IOMAP_DELALLOC:
- if (!buffer_uptodate(bh) ||
- (offset >= i_size_read(inode)))
- set_buffer_new(bh);
- set_buffer_uptodate(bh);
- set_buffer_mapped(bh);
- set_buffer_delay(bh);
- break;
- case IOMAP_UNWRITTEN:
-
- set_buffer_new(bh);
- set_buffer_unwritten(bh);
-
- case IOMAP_MAPPED:
- if (offset >= i_size_read(inode))
- set_buffer_new(bh);
- bh->b_blocknr = (iomap->blkno >> (inode->i_blkbits - 9)) +
- ((offset - iomap->offset) >> inode->i_blkbits);
- set_buffer_mapped(bh);
- break;
- }
- }
- int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
- get_block_t *get_block, struct iomap *iomap)
- {
- unsigned from = pos & (PAGE_SIZE - 1);
- unsigned to = from + len;
- struct inode *inode = page->mapping->host;
- unsigned block_start, block_end;
- sector_t block;
- int err = 0;
- unsigned blocksize, bbits;
- struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
- BUG_ON(!PageLocked(page));
- BUG_ON(from > PAGE_SIZE);
- BUG_ON(to > PAGE_SIZE);
- BUG_ON(from > to);
- head = create_page_buffers(page, inode, 0);
- blocksize = head->b_size;
- bbits = block_size_bits(blocksize);
- block = (sector_t)page->index << (PAGE_SHIFT - bbits);
- for(bh = head, block_start = 0; bh != head || !block_start;
- block++, block_start=block_end, bh = bh->b_this_page) {
- block_end = block_start + blocksize;
- if (block_end <= from || block_start >= to) {
- if (PageUptodate(page)) {
- if (!buffer_uptodate(bh))
- set_buffer_uptodate(bh);
- }
- continue;
- }
- if (buffer_new(bh))
- clear_buffer_new(bh);
- if (!buffer_mapped(bh)) {
- WARN_ON(bh->b_size != blocksize);
- if (get_block) {
- err = get_block(inode, block, bh, 1);
- if (err)
- break;
- } else {
- iomap_to_bh(inode, block, bh, iomap);
- }
- if (buffer_new(bh)) {
- unmap_underlying_metadata(bh->b_bdev,
- bh->b_blocknr);
- if (PageUptodate(page)) {
- clear_buffer_new(bh);
- set_buffer_uptodate(bh);
- mark_buffer_dirty(bh);
- continue;
- }
- if (block_end > to || block_start < from)
- zero_user_segments(page,
- to, block_end,
- block_start, from);
- continue;
- }
- }
- if (PageUptodate(page)) {
- if (!buffer_uptodate(bh))
- set_buffer_uptodate(bh);
- continue;
- }
- if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
- !buffer_unwritten(bh) &&
- (block_start < from || block_end > to)) {
- ll_rw_block(REQ_OP_READ, 0, 1, &bh);
- *wait_bh++=bh;
- }
- }
-
- while(wait_bh > wait) {
- wait_on_buffer(*--wait_bh);
- if (!buffer_uptodate(*wait_bh))
- err = -EIO;
- }
- if (unlikely(err))
- page_zero_new_buffers(page, from, to);
- return err;
- }
- int __block_write_begin(struct page *page, loff_t pos, unsigned len,
- get_block_t *get_block)
- {
- return __block_write_begin_int(page, pos, len, get_block, NULL);
- }
- EXPORT_SYMBOL(__block_write_begin);
- static int __block_commit_write(struct inode *inode, struct page *page,
- unsigned from, unsigned to)
- {
- unsigned block_start, block_end;
- int partial = 0;
- unsigned blocksize;
- struct buffer_head *bh, *head;
- bh = head = page_buffers(page);
- blocksize = bh->b_size;
- block_start = 0;
- do {
- block_end = block_start + blocksize;
- if (block_end <= from || block_start >= to) {
- if (!buffer_uptodate(bh))
- partial = 1;
- } else {
- set_buffer_uptodate(bh);
- mark_buffer_dirty(bh);
- }
- clear_buffer_new(bh);
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
-
- if (!partial)
- SetPageUptodate(page);
- return 0;
- }
- int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
- unsigned flags, struct page **pagep, get_block_t *get_block)
- {
- pgoff_t index = pos >> PAGE_SHIFT;
- struct page *page;
- int status;
- page = grab_cache_page_write_begin(mapping, index, flags);
- if (!page)
- return -ENOMEM;
- status = __block_write_begin(page, pos, len, get_block);
- if (unlikely(status)) {
- unlock_page(page);
- put_page(page);
- page = NULL;
- }
- *pagep = page;
- return status;
- }
- EXPORT_SYMBOL(block_write_begin);
- int block_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = mapping->host;
- unsigned start;
- start = pos & (PAGE_SIZE - 1);
- if (unlikely(copied < len)) {
-
- if (!PageUptodate(page))
- copied = 0;
- page_zero_new_buffers(page, start+copied, start+len);
- }
- flush_dcache_page(page);
-
- __block_commit_write(inode, page, start, start+copied);
- return copied;
- }
- EXPORT_SYMBOL(block_write_end);
- int generic_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = mapping->host;
- loff_t old_size = inode->i_size;
- int i_size_changed = 0;
- copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
-
- if (pos+copied > inode->i_size) {
- i_size_write(inode, pos+copied);
- i_size_changed = 1;
- }
- unlock_page(page);
- put_page(page);
- if (old_size < pos)
- pagecache_isize_extended(inode, old_size, pos);
-
- if (i_size_changed)
- mark_inode_dirty(inode);
- return copied;
- }
- EXPORT_SYMBOL(generic_write_end);
- int block_is_partially_uptodate(struct page *page, unsigned long from,
- unsigned long count)
- {
- unsigned block_start, block_end, blocksize;
- unsigned to;
- struct buffer_head *bh, *head;
- int ret = 1;
- if (!page_has_buffers(page))
- return 0;
- head = page_buffers(page);
- blocksize = head->b_size;
- to = min_t(unsigned, PAGE_SIZE - from, count);
- to = from + to;
- if (from < blocksize && to > PAGE_SIZE - blocksize)
- return 0;
- bh = head;
- block_start = 0;
- do {
- block_end = block_start + blocksize;
- if (block_end > from && block_start < to) {
- if (!buffer_uptodate(bh)) {
- ret = 0;
- break;
- }
- if (block_end >= to)
- break;
- }
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
- return ret;
- }
- EXPORT_SYMBOL(block_is_partially_uptodate);
- int block_read_full_page(struct page *page, get_block_t *get_block)
- {
- struct inode *inode = page->mapping->host;
- sector_t iblock, lblock;
- struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
- unsigned int blocksize, bbits;
- int nr, i;
- int fully_mapped = 1;
- head = create_page_buffers(page, inode, 0);
- blocksize = head->b_size;
- bbits = block_size_bits(blocksize);
- iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
- lblock = (i_size_read(inode)+blocksize-1) >> bbits;
- bh = head;
- nr = 0;
- i = 0;
- do {
- if (buffer_uptodate(bh))
- continue;
- if (!buffer_mapped(bh)) {
- int err = 0;
- fully_mapped = 0;
- if (iblock < lblock) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, iblock, bh, 0);
- if (err)
- SetPageError(page);
- }
- if (!buffer_mapped(bh)) {
- zero_user(page, i * blocksize, blocksize);
- if (!err)
- set_buffer_uptodate(bh);
- continue;
- }
-
- if (buffer_uptodate(bh))
- continue;
- }
- arr[nr++] = bh;
- } while (i++, iblock++, (bh = bh->b_this_page) != head);
- if (fully_mapped)
- SetPageMappedToDisk(page);
- if (!nr) {
-
- if (!PageError(page))
- SetPageUptodate(page);
- unlock_page(page);
- return 0;
- }
-
- for (i = 0; i < nr; i++) {
- bh = arr[i];
- lock_buffer(bh);
- mark_buffer_async_read(bh);
- }
-
- for (i = 0; i < nr; i++) {
- bh = arr[i];
- if (buffer_uptodate(bh))
- end_buffer_async_read(bh, 1);
- else
- submit_bh(REQ_OP_READ, 0, bh);
- }
- return 0;
- }
- EXPORT_SYMBOL(block_read_full_page);
- int generic_cont_expand_simple(struct inode *inode, loff_t size)
- {
- struct address_space *mapping = inode->i_mapping;
- struct page *page;
- void *fsdata;
- int err;
- err = inode_newsize_ok(inode, size);
- if (err)
- goto out;
- err = pagecache_write_begin(NULL, mapping, size, 0,
- AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
- &page, &fsdata);
- if (err)
- goto out;
- err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
- BUG_ON(err > 0);
- out:
- return err;
- }
- EXPORT_SYMBOL(generic_cont_expand_simple);
- static int cont_expand_zero(struct file *file, struct address_space *mapping,
- loff_t pos, loff_t *bytes)
- {
- struct inode *inode = mapping->host;
- unsigned int blocksize = i_blocksize(inode);
- struct page *page;
- void *fsdata;
- pgoff_t index, curidx;
- loff_t curpos;
- unsigned zerofrom, offset, len;
- int err = 0;
- index = pos >> PAGE_SHIFT;
- offset = pos & ~PAGE_MASK;
- while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
- zerofrom = curpos & ~PAGE_MASK;
- if (zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- len = PAGE_SIZE - zerofrom;
- err = pagecache_write_begin(file, mapping, curpos, len,
- AOP_FLAG_UNINTERRUPTIBLE,
- &page, &fsdata);
- if (err)
- goto out;
- zero_user(page, zerofrom, len);
- err = pagecache_write_end(file, mapping, curpos, len, len,
- page, fsdata);
- if (err < 0)
- goto out;
- BUG_ON(err != len);
- err = 0;
- balance_dirty_pages_ratelimited(mapping);
- if (unlikely(fatal_signal_pending(current))) {
- err = -EINTR;
- goto out;
- }
- }
-
- if (index == curidx) {
- zerofrom = curpos & ~PAGE_MASK;
-
- if (offset <= zerofrom) {
- goto out;
- }
- if (zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- len = offset - zerofrom;
- err = pagecache_write_begin(file, mapping, curpos, len,
- AOP_FLAG_UNINTERRUPTIBLE,
- &page, &fsdata);
- if (err)
- goto out;
- zero_user(page, zerofrom, len);
- err = pagecache_write_end(file, mapping, curpos, len, len,
- page, fsdata);
- if (err < 0)
- goto out;
- BUG_ON(err != len);
- err = 0;
- }
- out:
- return err;
- }
- int cont_write_begin(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned flags,
- struct page **pagep, void **fsdata,
- get_block_t *get_block, loff_t *bytes)
- {
- struct inode *inode = mapping->host;
- unsigned int blocksize = i_blocksize(inode);
- unsigned int zerofrom;
- int err;
- err = cont_expand_zero(file, mapping, pos, bytes);
- if (err)
- return err;
- zerofrom = *bytes & ~PAGE_MASK;
- if (pos+len > *bytes && zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- return block_write_begin(mapping, pos, len, flags, pagep, get_block);
- }
- EXPORT_SYMBOL(cont_write_begin);
- int block_commit_write(struct page *page, unsigned from, unsigned to)
- {
- struct inode *inode = page->mapping->host;
- __block_commit_write(inode,page,from,to);
- return 0;
- }
- EXPORT_SYMBOL(block_commit_write);
- int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
- get_block_t get_block)
- {
- struct page *page = vmf->page;
- struct inode *inode = file_inode(vma->vm_file);
- unsigned long end;
- loff_t size;
- int ret;
- lock_page(page);
- size = i_size_read(inode);
- if ((page->mapping != inode->i_mapping) ||
- (page_offset(page) > size)) {
-
- ret = -EFAULT;
- goto out_unlock;
- }
-
- if (((page->index + 1) << PAGE_SHIFT) > size)
- end = size & ~PAGE_MASK;
- else
- end = PAGE_SIZE;
- ret = __block_write_begin(page, 0, end, get_block);
- if (!ret)
- ret = block_commit_write(page, 0, end);
- if (unlikely(ret < 0))
- goto out_unlock;
- set_page_dirty(page);
- wait_for_stable_page(page);
- return 0;
- out_unlock:
- unlock_page(page);
- return ret;
- }
- EXPORT_SYMBOL(block_page_mkwrite);
- static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
- {
- __end_buffer_read_notouch(bh, uptodate);
- }
- static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
- {
- struct buffer_head *bh;
- BUG_ON(!PageLocked(page));
- spin_lock(&page->mapping->private_lock);
- bh = head;
- do {
- if (PageDirty(page))
- set_buffer_dirty(bh);
- if (!bh->b_this_page)
- bh->b_this_page = head;
- bh = bh->b_this_page;
- } while (bh != head);
- attach_page_buffers(page, head);
- spin_unlock(&page->mapping->private_lock);
- }
- int nobh_write_begin(struct address_space *mapping,
- loff_t pos, unsigned len, unsigned flags,
- struct page **pagep, void **fsdata,
- get_block_t *get_block)
- {
- struct inode *inode = mapping->host;
- const unsigned blkbits = inode->i_blkbits;
- const unsigned blocksize = 1 << blkbits;
- struct buffer_head *head, *bh;
- struct page *page;
- pgoff_t index;
- unsigned from, to;
- unsigned block_in_page;
- unsigned block_start, block_end;
- sector_t block_in_file;
- int nr_reads = 0;
- int ret = 0;
- int is_mapped_to_disk = 1;
- index = pos >> PAGE_SHIFT;
- from = pos & (PAGE_SIZE - 1);
- to = from + len;
- page = grab_cache_page_write_begin(mapping, index, flags);
- if (!page)
- return -ENOMEM;
- *pagep = page;
- *fsdata = NULL;
- if (page_has_buffers(page)) {
- ret = __block_write_begin(page, pos, len, get_block);
- if (unlikely(ret))
- goto out_release;
- return ret;
- }
- if (PageMappedToDisk(page))
- return 0;
-
- head = alloc_page_buffers(page, blocksize, 0);
- if (!head) {
- ret = -ENOMEM;
- goto out_release;
- }
- block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
-
- for (block_start = 0, block_in_page = 0, bh = head;
- block_start < PAGE_SIZE;
- block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
- int create;
- block_end = block_start + blocksize;
- bh->b_state = 0;
- create = 1;
- if (block_start >= to)
- create = 0;
- ret = get_block(inode, block_in_file + block_in_page,
- bh, create);
- if (ret)
- goto failed;
- if (!buffer_mapped(bh))
- is_mapped_to_disk = 0;
- if (buffer_new(bh))
- unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
- if (PageUptodate(page)) {
- set_buffer_uptodate(bh);
- continue;
- }
- if (buffer_new(bh) || !buffer_mapped(bh)) {
- zero_user_segments(page, block_start, from,
- to, block_end);
- continue;
- }
- if (buffer_uptodate(bh))
- continue;
- if (block_start < from || block_end > to) {
- lock_buffer(bh);
- bh->b_end_io = end_buffer_read_nobh;
- submit_bh(REQ_OP_READ, 0, bh);
- nr_reads++;
- }
- }
- if (nr_reads) {
-
- for (bh = head; bh; bh = bh->b_this_page) {
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- ret = -EIO;
- }
- if (ret)
- goto failed;
- }
- if (is_mapped_to_disk)
- SetPageMappedToDisk(page);
- *fsdata = head;
- return 0;
- failed:
- BUG_ON(!ret);
-
- attach_nobh_buffers(page, head);
- page_zero_new_buffers(page, from, to);
- out_release:
- unlock_page(page);
- put_page(page);
- *pagep = NULL;
- return ret;
- }
- EXPORT_SYMBOL(nobh_write_begin);
- int nobh_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = page->mapping->host;
- struct buffer_head *head = fsdata;
- struct buffer_head *bh;
- BUG_ON(fsdata != NULL && page_has_buffers(page));
- if (unlikely(copied < len) && head)
- attach_nobh_buffers(page, head);
- if (page_has_buffers(page))
- return generic_write_end(file, mapping, pos, len,
- copied, page, fsdata);
- SetPageUptodate(page);
- set_page_dirty(page);
- if (pos+copied > inode->i_size) {
- i_size_write(inode, pos+copied);
- mark_inode_dirty(inode);
- }
- unlock_page(page);
- put_page(page);
- while (head) {
- bh = head;
- head = head->b_this_page;
- free_buffer_head(bh);
- }
- return copied;
- }
- EXPORT_SYMBOL(nobh_write_end);
- int nobh_writepage(struct page *page, get_block_t *get_block,
- struct writeback_control *wbc)
- {
- struct inode * const inode = page->mapping->host;
- loff_t i_size = i_size_read(inode);
- const pgoff_t end_index = i_size >> PAGE_SHIFT;
- unsigned offset;
- int ret;
-
- if (page->index < end_index)
- goto out;
-
- offset = i_size & (PAGE_SIZE-1);
- if (page->index >= end_index+1 || !offset) {
-
- #if 0
-
- if (page->mapping->a_ops->invalidatepage)
- page->mapping->a_ops->invalidatepage(page, offset);
- #endif
- unlock_page(page);
- return 0;
- }
-
- zero_user_segment(page, offset, PAGE_SIZE);
- out:
- ret = mpage_writepage(page, get_block, wbc);
- if (ret == -EAGAIN)
- ret = __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
- return ret;
- }
- EXPORT_SYMBOL(nobh_writepage);
- int nobh_truncate_page(struct address_space *mapping,
- loff_t from, get_block_t *get_block)
- {
- pgoff_t index = from >> PAGE_SHIFT;
- unsigned offset = from & (PAGE_SIZE-1);
- unsigned blocksize;
- sector_t iblock;
- unsigned length, pos;
- struct inode *inode = mapping->host;
- struct page *page;
- struct buffer_head map_bh;
- int err;
- blocksize = i_blocksize(inode);
- length = offset & (blocksize - 1);
-
- if (!length)
- return 0;
- length = blocksize - length;
- iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
- page = grab_cache_page(mapping, index);
- err = -ENOMEM;
- if (!page)
- goto out;
- if (page_has_buffers(page)) {
- has_buffers:
- unlock_page(page);
- put_page(page);
- return block_truncate_page(mapping, from, get_block);
- }
-
- pos = blocksize;
- while (offset >= pos) {
- iblock++;
- pos += blocksize;
- }
- map_bh.b_size = blocksize;
- map_bh.b_state = 0;
- err = get_block(inode, iblock, &map_bh, 0);
- if (err)
- goto unlock;
-
- if (!buffer_mapped(&map_bh))
- goto unlock;
-
- if (!PageUptodate(page)) {
- err = mapping->a_ops->readpage(NULL, page);
- if (err) {
- put_page(page);
- goto out;
- }
- lock_page(page);
- if (!PageUptodate(page)) {
- err = -EIO;
- goto unlock;
- }
- if (page_has_buffers(page))
- goto has_buffers;
- }
- zero_user(page, offset, length);
- set_page_dirty(page);
- err = 0;
- unlock:
- unlock_page(page);
- put_page(page);
- out:
- return err;
- }
- EXPORT_SYMBOL(nobh_truncate_page);
- int block_truncate_page(struct address_space *mapping,
- loff_t from, get_block_t *get_block)
- {
- pgoff_t index = from >> PAGE_SHIFT;
- unsigned offset = from & (PAGE_SIZE-1);
- unsigned blocksize;
- sector_t iblock;
- unsigned length, pos;
- struct inode *inode = mapping->host;
- struct page *page;
- struct buffer_head *bh;
- int err;
- blocksize = i_blocksize(inode);
- length = offset & (blocksize - 1);
-
- if (!length)
- return 0;
- length = blocksize - length;
- iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
-
- page = grab_cache_page(mapping, index);
- err = -ENOMEM;
- if (!page)
- goto out;
- if (!page_has_buffers(page))
- create_empty_buffers(page, blocksize, 0);
-
- bh = page_buffers(page);
- pos = blocksize;
- while (offset >= pos) {
- bh = bh->b_this_page;
- iblock++;
- pos += blocksize;
- }
- err = 0;
- if (!buffer_mapped(bh)) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, iblock, bh, 0);
- if (err)
- goto unlock;
-
- if (!buffer_mapped(bh))
- goto unlock;
- }
-
- if (PageUptodate(page))
- set_buffer_uptodate(bh);
- if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
- err = -EIO;
- ll_rw_block(REQ_OP_READ, 0, 1, &bh);
- wait_on_buffer(bh);
-
- if (!buffer_uptodate(bh))
- goto unlock;
- }
- zero_user(page, offset, length);
- mark_buffer_dirty(bh);
- err = 0;
- unlock:
- unlock_page(page);
- put_page(page);
- out:
- return err;
- }
- EXPORT_SYMBOL(block_truncate_page);
- int block_write_full_page(struct page *page, get_block_t *get_block,
- struct writeback_control *wbc)
- {
- struct inode * const inode = page->mapping->host;
- loff_t i_size = i_size_read(inode);
- const pgoff_t end_index = i_size >> PAGE_SHIFT;
- unsigned offset;
-
- if (page->index < end_index)
- return __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
-
- offset = i_size & (PAGE_SIZE-1);
- if (page->index >= end_index+1 || !offset) {
-
- do_invalidatepage(page, 0, PAGE_SIZE);
- unlock_page(page);
- return 0;
- }
-
- zero_user_segment(page, offset, PAGE_SIZE);
- return __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
- }
- EXPORT_SYMBOL(block_write_full_page);
- sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
- get_block_t *get_block)
- {
- struct buffer_head tmp;
- struct inode *inode = mapping->host;
- tmp.b_state = 0;
- tmp.b_blocknr = 0;
- tmp.b_size = i_blocksize(inode);
- get_block(inode, block, &tmp, 0);
- return tmp.b_blocknr;
- }
- EXPORT_SYMBOL(generic_block_bmap);
- static void end_bio_bh_io_sync(struct bio *bio)
- {
- struct buffer_head *bh = bio->bi_private;
- if (unlikely(bio_flagged(bio, BIO_QUIET)))
- set_bit(BH_Quiet, &bh->b_state);
- bh->b_end_io(bh, !bio->bi_error);
- bio_put(bio);
- }
- void guard_bio_eod(int op, struct bio *bio)
- {
- sector_t maxsector;
- struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
- unsigned truncated_bytes;
- maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
- if (!maxsector)
- return;
-
- if (unlikely(bio->bi_iter.bi_sector >= maxsector))
- return;
- maxsector -= bio->bi_iter.bi_sector;
- if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
- return;
-
- truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
-
- if (truncated_bytes > bvec->bv_len)
- return;
-
- bio->bi_iter.bi_size -= truncated_bytes;
- bvec->bv_len -= truncated_bytes;
-
- if (op == REQ_OP_READ) {
- zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
- truncated_bytes);
- }
- }
- static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags, struct writeback_control *wbc)
- {
- struct bio *bio;
- BUG_ON(!buffer_locked(bh));
- BUG_ON(!buffer_mapped(bh));
- BUG_ON(!bh->b_end_io);
- BUG_ON(buffer_delay(bh));
- BUG_ON(buffer_unwritten(bh));
-
- if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
- clear_buffer_write_io_error(bh);
-
- bio = bio_alloc(GFP_NOIO, 1);
- if (wbc) {
- wbc_init_bio(wbc, bio);
- wbc_account_io(wbc, bh->b_page, bh->b_size);
- }
- bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
- bio->bi_bdev = bh->b_bdev;
- bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
- BUG_ON(bio->bi_iter.bi_size != bh->b_size);
- bio->bi_end_io = end_bio_bh_io_sync;
- bio->bi_private = bh;
- bio->bi_flags |= bio_flags;
-
- guard_bio_eod(op, bio);
- if (buffer_meta(bh))
- op_flags |= REQ_META;
- if (buffer_prio(bh))
- op_flags |= REQ_PRIO;
- bio_set_op_attrs(bio, op, op_flags);
- submit_bio(bio);
- return 0;
- }
- int _submit_bh(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags)
- {
- return submit_bh_wbc(op, op_flags, bh, bio_flags, NULL);
- }
- EXPORT_SYMBOL_GPL(_submit_bh);
- int submit_bh(int op, int op_flags, struct buffer_head *bh)
- {
- return submit_bh_wbc(op, op_flags, bh, 0, NULL);
- }
- EXPORT_SYMBOL(submit_bh);
- void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
- {
- int i;
- for (i = 0; i < nr; i++) {
- struct buffer_head *bh = bhs[i];
- if (!trylock_buffer(bh))
- continue;
- if (op == WRITE) {
- if (test_clear_buffer_dirty(bh)) {
- bh->b_end_io = end_buffer_write_sync;
- get_bh(bh);
- submit_bh(op, op_flags, bh);
- continue;
- }
- } else {
- if (!buffer_uptodate(bh)) {
- bh->b_end_io = end_buffer_read_sync;
- get_bh(bh);
- submit_bh(op, op_flags, bh);
- continue;
- }
- }
- unlock_buffer(bh);
- }
- }
- EXPORT_SYMBOL(ll_rw_block);
- void write_dirty_buffer(struct buffer_head *bh, int op_flags)
- {
- lock_buffer(bh);
- if (!test_clear_buffer_dirty(bh)) {
- unlock_buffer(bh);
- return;
- }
- bh->b_end_io = end_buffer_write_sync;
- get_bh(bh);
- submit_bh(REQ_OP_WRITE, op_flags, bh);
- }
- EXPORT_SYMBOL(write_dirty_buffer);
- int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
- {
- int ret = 0;
- WARN_ON(atomic_read(&bh->b_count) < 1);
- lock_buffer(bh);
- if (test_clear_buffer_dirty(bh)) {
- get_bh(bh);
- bh->b_end_io = end_buffer_write_sync;
- ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
- wait_on_buffer(bh);
- if (!ret && !buffer_uptodate(bh))
- ret = -EIO;
- } else {
- unlock_buffer(bh);
- }
- return ret;
- }
- EXPORT_SYMBOL(__sync_dirty_buffer);
- int sync_dirty_buffer(struct buffer_head *bh)
- {
- return __sync_dirty_buffer(bh, WRITE_SYNC);
- }
- EXPORT_SYMBOL(sync_dirty_buffer);
- static inline int buffer_busy(struct buffer_head *bh)
- {
- return atomic_read(&bh->b_count) |
- (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
- }
- static int
- drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
- {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh;
- bh = head;
- do {
- if (buffer_write_io_error(bh) && page->mapping)
- mapping_set_error(page->mapping, -EIO);
- if (buffer_busy(bh)) {
-
- evict_bh_lrus(bh);
- if (buffer_busy(bh))
- goto failed;
- }
- bh = bh->b_this_page;
- } while (bh != head);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (bh->b_assoc_map)
- __remove_assoc_queue(bh);
- bh = next;
- } while (bh != head);
- *buffers_to_free = head;
- __clear_page_buffers(page);
- return 1;
- failed:
- return 0;
- }
- int try_to_free_buffers(struct page *page)
- {
- struct address_space * const mapping = page->mapping;
- struct buffer_head *buffers_to_free = NULL;
- int ret = 0;
- BUG_ON(!PageLocked(page));
- if (PageWriteback(page))
- return 0;
- if (mapping == NULL) {
- ret = drop_buffers(page, &buffers_to_free);
- goto out;
- }
- spin_lock(&mapping->private_lock);
- ret = drop_buffers(page, &buffers_to_free);
-
- if (ret)
- cancel_dirty_page(page);
- spin_unlock(&mapping->private_lock);
- out:
- if (buffers_to_free) {
- struct buffer_head *bh = buffers_to_free;
- do {
- struct buffer_head *next = bh->b_this_page;
- free_buffer_head(bh);
- bh = next;
- } while (bh != buffers_to_free);
- }
- return ret;
- }
- EXPORT_SYMBOL(try_to_free_buffers);
- SYSCALL_DEFINE2(bdflush, int, func, long, data)
- {
- static int msg_count;
- if (!capable(CAP_SYS_ADMIN))
- return -EPERM;
- if (msg_count < 5) {
- msg_count++;
- printk(KERN_INFO
- "warning: process `%s' used the obsolete bdflush"
- " system call\n", current->comm);
- printk(KERN_INFO "Fix your initscripts?\n");
- }
- if (func == 1)
- do_exit(0);
- return 0;
- }
- static struct kmem_cache *bh_cachep __read_mostly;
- static unsigned long max_buffer_heads;
- int buffer_heads_over_limit;
- struct bh_accounting {
- int nr;
- int ratelimit;
- };
- static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
- static void recalc_bh_state(void)
- {
- int i;
- int tot = 0;
- if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
- return;
- __this_cpu_write(bh_accounting.ratelimit, 0);
- for_each_online_cpu(i)
- tot += per_cpu(bh_accounting, i).nr;
- buffer_heads_over_limit = (tot > max_buffer_heads);
- }
- struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
- {
- struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
- if (ret) {
- INIT_LIST_HEAD(&ret->b_assoc_buffers);
- preempt_disable();
- __this_cpu_inc(bh_accounting.nr);
- recalc_bh_state();
- preempt_enable();
- }
- return ret;
- }
- EXPORT_SYMBOL(alloc_buffer_head);
- void free_buffer_head(struct buffer_head *bh)
- {
- BUG_ON(!list_empty(&bh->b_assoc_buffers));
- kmem_cache_free(bh_cachep, bh);
- preempt_disable();
- __this_cpu_dec(bh_accounting.nr);
- recalc_bh_state();
- preempt_enable();
- }
- EXPORT_SYMBOL(free_buffer_head);
- static void buffer_exit_cpu(int cpu)
- {
- int i;
- struct bh_lru *b = &per_cpu(bh_lrus, cpu);
- for (i = 0; i < BH_LRU_SIZE; i++) {
- brelse(b->bhs[i]);
- b->bhs[i] = NULL;
- }
- this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
- per_cpu(bh_accounting, cpu).nr = 0;
- }
- static int buffer_cpu_notify(struct notifier_block *self,
- unsigned long action, void *hcpu)
- {
- if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
- buffer_exit_cpu((unsigned long)hcpu);
- return NOTIFY_OK;
- }
- int bh_uptodate_or_lock(struct buffer_head *bh)
- {
- if (!buffer_uptodate(bh)) {
- lock_buffer(bh);
- if (!buffer_uptodate(bh))
- return 0;
- unlock_buffer(bh);
- }
- return 1;
- }
- EXPORT_SYMBOL(bh_uptodate_or_lock);
- int bh_submit_read(struct buffer_head *bh)
- {
- BUG_ON(!buffer_locked(bh));
- if (buffer_uptodate(bh)) {
- unlock_buffer(bh);
- return 0;
- }
- get_bh(bh);
- bh->b_end_io = end_buffer_read_sync;
- submit_bh(REQ_OP_READ, 0, bh);
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
- return 0;
- return -EIO;
- }
- EXPORT_SYMBOL(bh_submit_read);
- void __init buffer_init(void)
- {
- unsigned long nrpages;
- bh_cachep = kmem_cache_create("buffer_head",
- sizeof(struct buffer_head), 0,
- (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
- SLAB_MEM_SPREAD),
- NULL);
-
- nrpages = (nr_free_buffer_pages() * 10) / 100;
- max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
- hotcpu_notifier(buffer_cpu_notify, 0);
- }
|