123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148 |
- #include <linux/export.h>
- #include <linux/fs.h>
- #include <linux/mm.h>
- #include <linux/backing-dev.h>
- #include <linux/hash.h>
- #include <linux/swap.h>
- #include <linux/security.h>
- #include <linux/cdev.h>
- #include <linux/bootmem.h>
- #include <linux/fsnotify.h>
- #include <linux/mount.h>
- #include <linux/posix_acl.h>
- #include <linux/prefetch.h>
- #include <linux/buffer_head.h> /* for inode_has_buffers */
- #include <linux/ratelimit.h>
- #include <linux/list_lru.h>
- #include <trace/events/writeback.h>
- #include "internal.h"
- static unsigned int i_hash_mask __read_mostly;
- static unsigned int i_hash_shift __read_mostly;
- static struct hlist_head *inode_hashtable __read_mostly;
- static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
- const struct address_space_operations empty_aops = {
- };
- EXPORT_SYMBOL(empty_aops);
- struct inodes_stat_t inodes_stat;
- static DEFINE_PER_CPU(unsigned long, nr_inodes);
- static DEFINE_PER_CPU(unsigned long, nr_unused);
- static struct kmem_cache *inode_cachep __read_mostly;
- static long get_nr_inodes(void)
- {
- int i;
- long sum = 0;
- for_each_possible_cpu(i)
- sum += per_cpu(nr_inodes, i);
- return sum < 0 ? 0 : sum;
- }
- static inline long get_nr_inodes_unused(void)
- {
- int i;
- long sum = 0;
- for_each_possible_cpu(i)
- sum += per_cpu(nr_unused, i);
- return sum < 0 ? 0 : sum;
- }
- long get_nr_dirty_inodes(void)
- {
-
- long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
- return nr_dirty > 0 ? nr_dirty : 0;
- }
- #ifdef CONFIG_SYSCTL
- int proc_nr_inodes(struct ctl_table *table, int write,
- void __user *buffer, size_t *lenp, loff_t *ppos)
- {
- inodes_stat.nr_inodes = get_nr_inodes();
- inodes_stat.nr_unused = get_nr_inodes_unused();
- return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
- }
- #endif
- static int no_open(struct inode *inode, struct file *file)
- {
- return -ENXIO;
- }
- int inode_init_always(struct super_block *sb, struct inode *inode)
- {
- static const struct inode_operations empty_iops;
- static const struct file_operations no_open_fops = {.open = no_open};
- struct address_space *const mapping = &inode->i_data;
- inode->i_sb = sb;
- inode->i_blkbits = sb->s_blocksize_bits;
- inode->i_flags = 0;
- atomic_set(&inode->i_count, 1);
- inode->i_op = &empty_iops;
- inode->i_fop = &no_open_fops;
- inode->__i_nlink = 1;
- inode->i_opflags = 0;
- if (sb->s_xattr)
- inode->i_opflags |= IOP_XATTR;
- i_uid_write(inode, 0);
- i_gid_write(inode, 0);
- atomic_set(&inode->i_writecount, 0);
- inode->i_size = 0;
- inode->i_blocks = 0;
- inode->i_bytes = 0;
- inode->i_generation = 0;
- inode->i_pipe = NULL;
- inode->i_bdev = NULL;
- inode->i_cdev = NULL;
- inode->i_link = NULL;
- inode->i_dir_seq = 0;
- inode->i_rdev = 0;
- inode->dirtied_when = 0;
- #ifdef CONFIG_CGROUP_WRITEBACK
- inode->i_wb_frn_winner = 0;
- inode->i_wb_frn_avg_time = 0;
- inode->i_wb_frn_history = 0;
- #endif
- if (security_inode_alloc(inode))
- goto out;
- spin_lock_init(&inode->i_lock);
- lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
- init_rwsem(&inode->i_rwsem);
- lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
- atomic_set(&inode->i_dio_count, 0);
- mapping->a_ops = &empty_aops;
- mapping->host = inode;
- mapping->flags = 0;
- atomic_set(&mapping->i_mmap_writable, 0);
- mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
- mapping->private_data = NULL;
- mapping->writeback_index = 0;
- inode->i_private = NULL;
- inode->i_mapping = mapping;
- INIT_HLIST_HEAD(&inode->i_dentry);
- #ifdef CONFIG_FS_POSIX_ACL
- inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
- #endif
- #ifdef CONFIG_FSNOTIFY
- inode->i_fsnotify_mask = 0;
- #endif
- inode->i_flctx = NULL;
- this_cpu_inc(nr_inodes);
- return 0;
- out:
- return -ENOMEM;
- }
- EXPORT_SYMBOL(inode_init_always);
- static struct inode *alloc_inode(struct super_block *sb)
- {
- struct inode *inode;
- if (sb->s_op->alloc_inode)
- inode = sb->s_op->alloc_inode(sb);
- else
- inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
- if (!inode)
- return NULL;
- if (unlikely(inode_init_always(sb, inode))) {
- if (inode->i_sb->s_op->destroy_inode)
- inode->i_sb->s_op->destroy_inode(inode);
- else
- kmem_cache_free(inode_cachep, inode);
- return NULL;
- }
- return inode;
- }
- void free_inode_nonrcu(struct inode *inode)
- {
- kmem_cache_free(inode_cachep, inode);
- }
- EXPORT_SYMBOL(free_inode_nonrcu);
- void __destroy_inode(struct inode *inode)
- {
- BUG_ON(inode_has_buffers(inode));
- inode_detach_wb(inode);
- security_inode_free(inode);
- fsnotify_inode_delete(inode);
- locks_free_lock_context(inode);
- if (!inode->i_nlink) {
- WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
- atomic_long_dec(&inode->i_sb->s_remove_count);
- }
- #ifdef CONFIG_FS_POSIX_ACL
- if (inode->i_acl && !is_uncached_acl(inode->i_acl))
- posix_acl_release(inode->i_acl);
- if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
- posix_acl_release(inode->i_default_acl);
- #endif
- this_cpu_dec(nr_inodes);
- }
- EXPORT_SYMBOL(__destroy_inode);
- static void i_callback(struct rcu_head *head)
- {
- struct inode *inode = container_of(head, struct inode, i_rcu);
- kmem_cache_free(inode_cachep, inode);
- }
- static void destroy_inode(struct inode *inode)
- {
- BUG_ON(!list_empty(&inode->i_lru));
- __destroy_inode(inode);
- if (inode->i_sb->s_op->destroy_inode)
- inode->i_sb->s_op->destroy_inode(inode);
- else
- call_rcu(&inode->i_rcu, i_callback);
- }
- void drop_nlink(struct inode *inode)
- {
- WARN_ON(inode->i_nlink == 0);
- inode->__i_nlink--;
- if (!inode->i_nlink) {
- atomic_long_inc(&inode->i_sb->s_remove_count);
- #if IS_ENABLED(CONFIG_FS_VERITY)
- if (!list_empty(&inode->i_fsverity_list)) {
- spin_lock(&inode->i_sb->s_inode_fsveritylist_lock);
- list_del_init(&inode->i_fsverity_list);
- spin_unlock(&inode->i_sb->s_inode_fsveritylist_lock);
- iput(inode);
- }
- #endif
- }
- }
- EXPORT_SYMBOL(drop_nlink);
- void clear_nlink(struct inode *inode)
- {
- if (inode->i_nlink) {
- inode->__i_nlink = 0;
- atomic_long_inc(&inode->i_sb->s_remove_count);
- }
- }
- EXPORT_SYMBOL(clear_nlink);
- void set_nlink(struct inode *inode, unsigned int nlink)
- {
- if (!nlink) {
- clear_nlink(inode);
- } else {
-
- if (inode->i_nlink == 0)
- atomic_long_dec(&inode->i_sb->s_remove_count);
- inode->__i_nlink = nlink;
- }
- }
- EXPORT_SYMBOL(set_nlink);
- void inc_nlink(struct inode *inode)
- {
- if (unlikely(inode->i_nlink == 0)) {
- WARN_ON(!(inode->i_state & I_LINKABLE));
- atomic_long_dec(&inode->i_sb->s_remove_count);
- }
- inode->__i_nlink++;
- }
- EXPORT_SYMBOL(inc_nlink);
- void address_space_init_once(struct address_space *mapping)
- {
- memset(mapping, 0, sizeof(*mapping));
- INIT_RADIX_TREE(&mapping->page_tree, GFP_ATOMIC | __GFP_ACCOUNT);
- spin_lock_init(&mapping->tree_lock);
- init_rwsem(&mapping->i_mmap_rwsem);
- INIT_LIST_HEAD(&mapping->private_list);
- spin_lock_init(&mapping->private_lock);
- mapping->i_mmap = RB_ROOT;
- }
- EXPORT_SYMBOL(address_space_init_once);
- void inode_init_once(struct inode *inode)
- {
- memset(inode, 0, sizeof(*inode));
- INIT_HLIST_NODE(&inode->i_hash);
- INIT_LIST_HEAD(&inode->i_devices);
- INIT_LIST_HEAD(&inode->i_io_list);
- INIT_LIST_HEAD(&inode->i_wb_list);
- INIT_LIST_HEAD(&inode->i_lru);
- #if IS_ENABLED(CONFIG_FS_VERITY)
- INIT_LIST_HEAD(&inode->i_fsverity_list);
- #endif
- address_space_init_once(&inode->i_data);
- i_size_ordered_init(inode);
- #ifdef CONFIG_FSNOTIFY
- INIT_HLIST_HEAD(&inode->i_fsnotify_marks);
- #endif
- }
- EXPORT_SYMBOL(inode_init_once);
- static void init_once(void *foo)
- {
- struct inode *inode = (struct inode *) foo;
- inode_init_once(inode);
- }
- void __iget(struct inode *inode)
- {
- atomic_inc(&inode->i_count);
- }
- void ihold(struct inode *inode)
- {
- WARN_ON(atomic_inc_return(&inode->i_count) < 2);
- }
- EXPORT_SYMBOL(ihold);
- static void inode_lru_list_add(struct inode *inode)
- {
- if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
- this_cpu_inc(nr_unused);
- }
- void inode_add_lru(struct inode *inode)
- {
- if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
- I_FREEING | I_WILL_FREE)) &&
- !atomic_read(&inode->i_count) && inode->i_sb->s_flags & MS_ACTIVE)
- inode_lru_list_add(inode);
- }
- static void inode_lru_list_del(struct inode *inode)
- {
- if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
- this_cpu_dec(nr_unused);
- }
- void inode_sb_list_add(struct inode *inode)
- {
- spin_lock(&inode->i_sb->s_inode_list_lock);
- list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
- spin_unlock(&inode->i_sb->s_inode_list_lock);
- }
- EXPORT_SYMBOL_GPL(inode_sb_list_add);
- static inline void inode_sb_list_del(struct inode *inode)
- {
- if (!list_empty(&inode->i_sb_list)) {
- spin_lock(&inode->i_sb->s_inode_list_lock);
- list_del_init(&inode->i_sb_list);
- spin_unlock(&inode->i_sb->s_inode_list_lock);
- }
- }
- static unsigned long hash(struct super_block *sb, unsigned long hashval)
- {
- unsigned long tmp;
- tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
- L1_CACHE_BYTES;
- tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
- return tmp & i_hash_mask;
- }
- void __insert_inode_hash(struct inode *inode, unsigned long hashval)
- {
- struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
- spin_lock(&inode_hash_lock);
- spin_lock(&inode->i_lock);
- hlist_add_head(&inode->i_hash, b);
- spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
- }
- EXPORT_SYMBOL(__insert_inode_hash);
- void __remove_inode_hash(struct inode *inode)
- {
- spin_lock(&inode_hash_lock);
- spin_lock(&inode->i_lock);
- hlist_del_init(&inode->i_hash);
- spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
- }
- EXPORT_SYMBOL(__remove_inode_hash);
- void clear_inode(struct inode *inode)
- {
- might_sleep();
-
- spin_lock_irq(&inode->i_data.tree_lock);
- BUG_ON(inode->i_data.nrpages);
- BUG_ON(inode->i_data.nrexceptional);
- spin_unlock_irq(&inode->i_data.tree_lock);
- BUG_ON(!list_empty(&inode->i_data.private_list));
- BUG_ON(!(inode->i_state & I_FREEING));
- BUG_ON(inode->i_state & I_CLEAR);
- BUG_ON(!list_empty(&inode->i_wb_list));
-
- inode->i_state = I_FREEING | I_CLEAR;
- }
- EXPORT_SYMBOL(clear_inode);
- static void evict(struct inode *inode)
- {
- const struct super_operations *op = inode->i_sb->s_op;
- BUG_ON(!(inode->i_state & I_FREEING));
- BUG_ON(!list_empty(&inode->i_lru));
- if (!list_empty(&inode->i_io_list))
- inode_io_list_del(inode);
- inode_sb_list_del(inode);
-
- inode_wait_for_writeback(inode);
- if (op->evict_inode) {
- op->evict_inode(inode);
- } else {
- truncate_inode_pages_final(&inode->i_data);
- clear_inode(inode);
- }
- if (S_ISBLK(inode->i_mode) && inode->i_bdev)
- bd_forget(inode);
- if (S_ISCHR(inode->i_mode) && inode->i_cdev)
- cd_forget(inode);
- remove_inode_hash(inode);
- spin_lock(&inode->i_lock);
- wake_up_bit(&inode->i_state, __I_NEW);
- BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
- spin_unlock(&inode->i_lock);
- destroy_inode(inode);
- }
- static void dispose_list(struct list_head *head)
- {
- while (!list_empty(head)) {
- struct inode *inode;
- inode = list_first_entry(head, struct inode, i_lru);
- list_del_init(&inode->i_lru);
- evict(inode);
- cond_resched();
- }
- }
- void evict_inodes(struct super_block *sb)
- {
- struct inode *inode, *next;
- LIST_HEAD(dispose);
- again:
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
- if (atomic_read(&inode->i_count))
- continue;
- spin_lock(&inode->i_lock);
- if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
- spin_unlock(&inode->i_lock);
- continue;
- }
- inode->i_state |= I_FREEING;
- inode_lru_list_del(inode);
- spin_unlock(&inode->i_lock);
- list_add(&inode->i_lru, &dispose);
-
- if (need_resched()) {
- spin_unlock(&sb->s_inode_list_lock);
- cond_resched();
- dispose_list(&dispose);
- goto again;
- }
- }
- spin_unlock(&sb->s_inode_list_lock);
- dispose_list(&dispose);
- }
- EXPORT_SYMBOL_GPL(evict_inodes);
- int invalidate_inodes(struct super_block *sb, bool kill_dirty)
- {
- int busy = 0;
- struct inode *inode, *next;
- LIST_HEAD(dispose);
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
- spin_lock(&inode->i_lock);
- if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
- spin_unlock(&inode->i_lock);
- continue;
- }
- if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
- spin_unlock(&inode->i_lock);
- busy = 1;
- continue;
- }
- if (atomic_read(&inode->i_count)) {
- spin_unlock(&inode->i_lock);
- busy = 1;
- continue;
- }
- inode->i_state |= I_FREEING;
- inode_lru_list_del(inode);
- spin_unlock(&inode->i_lock);
- list_add(&inode->i_lru, &dispose);
- }
- spin_unlock(&sb->s_inode_list_lock);
- dispose_list(&dispose);
- return busy;
- }
- static enum lru_status inode_lru_isolate(struct list_head *item,
- struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
- {
- struct list_head *freeable = arg;
- struct inode *inode = container_of(item, struct inode, i_lru);
-
- if (!spin_trylock(&inode->i_lock))
- return LRU_SKIP;
-
- if (atomic_read(&inode->i_count) ||
- (inode->i_state & ~I_REFERENCED)) {
- list_lru_isolate(lru, &inode->i_lru);
- spin_unlock(&inode->i_lock);
- this_cpu_dec(nr_unused);
- return LRU_REMOVED;
- }
-
- if (inode->i_state & I_REFERENCED) {
- inode->i_state &= ~I_REFERENCED;
- spin_unlock(&inode->i_lock);
- return LRU_ROTATE;
- }
- if (inode_has_buffers(inode) || inode->i_data.nrpages) {
- __iget(inode);
- spin_unlock(&inode->i_lock);
- spin_unlock(lru_lock);
- if (remove_inode_buffers(inode)) {
- unsigned long reap;
- reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
- if (current_is_kswapd())
- __count_vm_events(KSWAPD_INODESTEAL, reap);
- else
- __count_vm_events(PGINODESTEAL, reap);
- if (current->reclaim_state)
- current->reclaim_state->reclaimed_slab += reap;
- }
- iput(inode);
- spin_lock(lru_lock);
- return LRU_RETRY;
- }
- WARN_ON(inode->i_state & I_NEW);
- inode->i_state |= I_FREEING;
- list_lru_isolate_move(lru, &inode->i_lru, freeable);
- spin_unlock(&inode->i_lock);
- this_cpu_dec(nr_unused);
- return LRU_REMOVED;
- }
- long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
- {
- LIST_HEAD(freeable);
- long freed;
- freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
- inode_lru_isolate, &freeable);
- dispose_list(&freeable);
- return freed;
- }
- static void __wait_on_freeing_inode(struct inode *inode);
- static struct inode *find_inode(struct super_block *sb,
- struct hlist_head *head,
- int (*test)(struct inode *, void *),
- void *data)
- {
- struct inode *inode = NULL;
- repeat:
- hlist_for_each_entry(inode, head, i_hash) {
- if (inode->i_sb != sb)
- continue;
- if (!test(inode, data))
- continue;
- spin_lock(&inode->i_lock);
- if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
- __wait_on_freeing_inode(inode);
- goto repeat;
- }
- __iget(inode);
- spin_unlock(&inode->i_lock);
- return inode;
- }
- return NULL;
- }
- static struct inode *find_inode_fast(struct super_block *sb,
- struct hlist_head *head, unsigned long ino)
- {
- struct inode *inode = NULL;
- repeat:
- hlist_for_each_entry(inode, head, i_hash) {
- if (inode->i_ino != ino)
- continue;
- if (inode->i_sb != sb)
- continue;
- spin_lock(&inode->i_lock);
- if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
- __wait_on_freeing_inode(inode);
- goto repeat;
- }
- __iget(inode);
- spin_unlock(&inode->i_lock);
- return inode;
- }
- return NULL;
- }
- #define LAST_INO_BATCH 1024
- static DEFINE_PER_CPU(unsigned int, last_ino);
- unsigned int get_next_ino(void)
- {
- unsigned int *p = &get_cpu_var(last_ino);
- unsigned int res = *p;
- #ifdef CONFIG_SMP
- if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
- static atomic_t shared_last_ino;
- int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
- res = next - LAST_INO_BATCH;
- }
- #endif
- res++;
-
- if (unlikely(!res))
- res++;
- *p = res;
- put_cpu_var(last_ino);
- return res;
- }
- EXPORT_SYMBOL(get_next_ino);
- struct inode *new_inode_pseudo(struct super_block *sb)
- {
- struct inode *inode = alloc_inode(sb);
- if (inode) {
- spin_lock(&inode->i_lock);
- inode->i_state = 0;
- spin_unlock(&inode->i_lock);
- INIT_LIST_HEAD(&inode->i_sb_list);
- }
- return inode;
- }
- struct inode *new_inode(struct super_block *sb)
- {
- struct inode *inode;
- spin_lock_prefetch(&sb->s_inode_list_lock);
- inode = new_inode_pseudo(sb);
- if (inode)
- inode_sb_list_add(inode);
- return inode;
- }
- EXPORT_SYMBOL(new_inode);
- #ifdef CONFIG_DEBUG_LOCK_ALLOC
- void lockdep_annotate_inode_mutex_key(struct inode *inode)
- {
- if (S_ISDIR(inode->i_mode)) {
- struct file_system_type *type = inode->i_sb->s_type;
-
- if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
-
-
- init_rwsem(&inode->i_rwsem);
- lockdep_set_class(&inode->i_rwsem,
- &type->i_mutex_dir_key);
- }
- }
- }
- EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
- #endif
- void unlock_new_inode(struct inode *inode)
- {
- lockdep_annotate_inode_mutex_key(inode);
- spin_lock(&inode->i_lock);
- WARN_ON(!(inode->i_state & I_NEW));
- inode->i_state &= ~I_NEW;
- smp_mb();
- wake_up_bit(&inode->i_state, __I_NEW);
- spin_unlock(&inode->i_lock);
- }
- EXPORT_SYMBOL(unlock_new_inode);
- void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
- {
- if (inode1 > inode2)
- swap(inode1, inode2);
- if (inode1 && !S_ISDIR(inode1->i_mode))
- inode_lock(inode1);
- if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
- inode_lock_nested(inode2, I_MUTEX_NONDIR2);
- }
- EXPORT_SYMBOL(lock_two_nondirectories);
- void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
- {
- if (inode1 && !S_ISDIR(inode1->i_mode))
- inode_unlock(inode1);
- if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
- inode_unlock(inode2);
- }
- EXPORT_SYMBOL(unlock_two_nondirectories);
- struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
- int (*test)(struct inode *, void *),
- int (*set)(struct inode *, void *), void *data)
- {
- struct hlist_head *head = inode_hashtable + hash(sb, hashval);
- struct inode *inode;
- again:
- spin_lock(&inode_hash_lock);
- inode = find_inode(sb, head, test, data);
- spin_unlock(&inode_hash_lock);
- if (inode) {
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- return inode;
- }
- inode = alloc_inode(sb);
- if (inode) {
- struct inode *old;
- spin_lock(&inode_hash_lock);
-
- old = find_inode(sb, head, test, data);
- if (!old) {
- if (set(inode, data))
- goto set_failed;
- spin_lock(&inode->i_lock);
- inode->i_state = I_NEW;
- hlist_add_head(&inode->i_hash, head);
- spin_unlock(&inode->i_lock);
- inode_sb_list_add(inode);
- spin_unlock(&inode_hash_lock);
-
- return inode;
- }
-
- spin_unlock(&inode_hash_lock);
- destroy_inode(inode);
- inode = old;
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- }
- return inode;
- set_failed:
- spin_unlock(&inode_hash_lock);
- destroy_inode(inode);
- return NULL;
- }
- EXPORT_SYMBOL(iget5_locked);
- struct inode *iget_locked(struct super_block *sb, unsigned long ino)
- {
- struct hlist_head *head = inode_hashtable + hash(sb, ino);
- struct inode *inode;
- again:
- spin_lock(&inode_hash_lock);
- inode = find_inode_fast(sb, head, ino);
- spin_unlock(&inode_hash_lock);
- if (inode) {
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- return inode;
- }
- inode = alloc_inode(sb);
- if (inode) {
- struct inode *old;
- spin_lock(&inode_hash_lock);
-
- old = find_inode_fast(sb, head, ino);
- if (!old) {
- inode->i_ino = ino;
- spin_lock(&inode->i_lock);
- inode->i_state = I_NEW;
- hlist_add_head(&inode->i_hash, head);
- spin_unlock(&inode->i_lock);
- inode_sb_list_add(inode);
- spin_unlock(&inode_hash_lock);
-
- return inode;
- }
-
- spin_unlock(&inode_hash_lock);
- destroy_inode(inode);
- inode = old;
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- }
- return inode;
- }
- EXPORT_SYMBOL(iget_locked);
- static int test_inode_iunique(struct super_block *sb, unsigned long ino)
- {
- struct hlist_head *b = inode_hashtable + hash(sb, ino);
- struct inode *inode;
- spin_lock(&inode_hash_lock);
- hlist_for_each_entry(inode, b, i_hash) {
- if (inode->i_ino == ino && inode->i_sb == sb) {
- spin_unlock(&inode_hash_lock);
- return 0;
- }
- }
- spin_unlock(&inode_hash_lock);
- return 1;
- }
- ino_t iunique(struct super_block *sb, ino_t max_reserved)
- {
-
- static DEFINE_SPINLOCK(iunique_lock);
- static unsigned int counter;
- ino_t res;
- spin_lock(&iunique_lock);
- do {
- if (counter <= max_reserved)
- counter = max_reserved + 1;
- res = counter++;
- } while (!test_inode_iunique(sb, res));
- spin_unlock(&iunique_lock);
- return res;
- }
- EXPORT_SYMBOL(iunique);
- struct inode *igrab(struct inode *inode)
- {
- spin_lock(&inode->i_lock);
- if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
- __iget(inode);
- spin_unlock(&inode->i_lock);
- } else {
- spin_unlock(&inode->i_lock);
-
- inode = NULL;
- }
- return inode;
- }
- EXPORT_SYMBOL(igrab);
- struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
- int (*test)(struct inode *, void *), void *data)
- {
- struct hlist_head *head = inode_hashtable + hash(sb, hashval);
- struct inode *inode;
- spin_lock(&inode_hash_lock);
- inode = find_inode(sb, head, test, data);
- spin_unlock(&inode_hash_lock);
- return inode;
- }
- EXPORT_SYMBOL(ilookup5_nowait);
- struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
- int (*test)(struct inode *, void *), void *data)
- {
- struct inode *inode;
- again:
- inode = ilookup5_nowait(sb, hashval, test, data);
- if (inode) {
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- }
- return inode;
- }
- EXPORT_SYMBOL(ilookup5);
- struct inode *ilookup(struct super_block *sb, unsigned long ino)
- {
- struct hlist_head *head = inode_hashtable + hash(sb, ino);
- struct inode *inode;
- again:
- spin_lock(&inode_hash_lock);
- inode = find_inode_fast(sb, head, ino);
- spin_unlock(&inode_hash_lock);
- if (inode) {
- wait_on_inode(inode);
- if (unlikely(inode_unhashed(inode))) {
- iput(inode);
- goto again;
- }
- }
- return inode;
- }
- EXPORT_SYMBOL(ilookup);
- struct inode *find_inode_nowait(struct super_block *sb,
- unsigned long hashval,
- int (*match)(struct inode *, unsigned long,
- void *),
- void *data)
- {
- struct hlist_head *head = inode_hashtable + hash(sb, hashval);
- struct inode *inode, *ret_inode = NULL;
- int mval;
- spin_lock(&inode_hash_lock);
- hlist_for_each_entry(inode, head, i_hash) {
- if (inode->i_sb != sb)
- continue;
- mval = match(inode, hashval, data);
- if (mval == 0)
- continue;
- if (mval == 1)
- ret_inode = inode;
- goto out;
- }
- out:
- spin_unlock(&inode_hash_lock);
- return ret_inode;
- }
- EXPORT_SYMBOL(find_inode_nowait);
- int insert_inode_locked(struct inode *inode)
- {
- struct super_block *sb = inode->i_sb;
- ino_t ino = inode->i_ino;
- struct hlist_head *head = inode_hashtable + hash(sb, ino);
- while (1) {
- struct inode *old = NULL;
- spin_lock(&inode_hash_lock);
- hlist_for_each_entry(old, head, i_hash) {
- if (old->i_ino != ino)
- continue;
- if (old->i_sb != sb)
- continue;
- spin_lock(&old->i_lock);
- if (old->i_state & (I_FREEING|I_WILL_FREE)) {
- spin_unlock(&old->i_lock);
- continue;
- }
- break;
- }
- if (likely(!old)) {
- spin_lock(&inode->i_lock);
- inode->i_state |= I_NEW;
- hlist_add_head(&inode->i_hash, head);
- spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
- return 0;
- }
- __iget(old);
- spin_unlock(&old->i_lock);
- spin_unlock(&inode_hash_lock);
- wait_on_inode(old);
- if (unlikely(!inode_unhashed(old))) {
- iput(old);
- return -EBUSY;
- }
- iput(old);
- }
- }
- EXPORT_SYMBOL(insert_inode_locked);
- int insert_inode_locked4(struct inode *inode, unsigned long hashval,
- int (*test)(struct inode *, void *), void *data)
- {
- struct super_block *sb = inode->i_sb;
- struct hlist_head *head = inode_hashtable + hash(sb, hashval);
- while (1) {
- struct inode *old = NULL;
- spin_lock(&inode_hash_lock);
- hlist_for_each_entry(old, head, i_hash) {
- if (old->i_sb != sb)
- continue;
- if (!test(old, data))
- continue;
- spin_lock(&old->i_lock);
- if (old->i_state & (I_FREEING|I_WILL_FREE)) {
- spin_unlock(&old->i_lock);
- continue;
- }
- break;
- }
- if (likely(!old)) {
- spin_lock(&inode->i_lock);
- inode->i_state |= I_NEW;
- hlist_add_head(&inode->i_hash, head);
- spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
- return 0;
- }
- __iget(old);
- spin_unlock(&old->i_lock);
- spin_unlock(&inode_hash_lock);
- wait_on_inode(old);
- if (unlikely(!inode_unhashed(old))) {
- iput(old);
- return -EBUSY;
- }
- iput(old);
- }
- }
- EXPORT_SYMBOL(insert_inode_locked4);
- int generic_delete_inode(struct inode *inode)
- {
- return 1;
- }
- EXPORT_SYMBOL(generic_delete_inode);
- static void iput_final(struct inode *inode)
- {
- struct super_block *sb = inode->i_sb;
- const struct super_operations *op = inode->i_sb->s_op;
- int drop;
- WARN_ON(inode->i_state & I_NEW);
- if (op->drop_inode)
- drop = op->drop_inode(inode);
- else
- drop = generic_drop_inode(inode);
- if (!drop && (sb->s_flags & MS_ACTIVE)) {
- inode->i_state |= I_REFERENCED;
- inode_add_lru(inode);
- spin_unlock(&inode->i_lock);
- return;
- }
- if (!drop) {
- inode->i_state |= I_WILL_FREE;
- spin_unlock(&inode->i_lock);
- write_inode_now(inode, 1);
- spin_lock(&inode->i_lock);
- WARN_ON(inode->i_state & I_NEW);
- inode->i_state &= ~I_WILL_FREE;
- }
- inode->i_state |= I_FREEING;
- if (!list_empty(&inode->i_lru))
- inode_lru_list_del(inode);
- spin_unlock(&inode->i_lock);
- evict(inode);
- }
- void iput(struct inode *inode)
- {
- if (!inode)
- return;
- BUG_ON(inode->i_state & I_CLEAR);
- retry:
- if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
- if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
- atomic_inc(&inode->i_count);
- inode->i_state &= ~I_DIRTY_TIME;
- spin_unlock(&inode->i_lock);
- trace_writeback_lazytime_iput(inode);
- mark_inode_dirty_sync(inode);
- goto retry;
- }
- iput_final(inode);
- }
- }
- EXPORT_SYMBOL(iput);
- sector_t bmap(struct inode *inode, sector_t block)
- {
- sector_t res = 0;
- if (inode->i_mapping->a_ops->bmap)
- res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
- return res;
- }
- EXPORT_SYMBOL(bmap);
- static void update_ovl_inode_times(struct dentry *dentry, struct inode *inode,
- bool rcu)
- {
- if (!rcu) {
- struct inode *realinode = d_real_inode(dentry);
- if (unlikely(inode != realinode) &&
- (!timespec_equal(&inode->i_mtime, &realinode->i_mtime) ||
- !timespec_equal(&inode->i_ctime, &realinode->i_ctime))) {
- inode->i_mtime = realinode->i_mtime;
- inode->i_ctime = realinode->i_ctime;
- }
- }
- }
- static int relatime_need_update(const struct path *path, struct inode *inode,
- struct timespec now, bool rcu)
- {
- if (!(path->mnt->mnt_flags & MNT_RELATIME))
- return 1;
- update_ovl_inode_times(path->dentry, inode, rcu);
-
- if (timespec_compare(&inode->i_mtime, &inode->i_atime) >= 0)
- return 1;
-
- if (timespec_compare(&inode->i_ctime, &inode->i_atime) >= 0)
- return 1;
-
- if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
- return 1;
-
- return 0;
- }
- int generic_update_time(struct inode *inode, struct timespec *time, int flags)
- {
- int iflags = I_DIRTY_TIME;
- if (flags & S_ATIME)
- inode->i_atime = *time;
- if (flags & S_VERSION)
- inode_inc_iversion(inode);
- if (flags & S_CTIME)
- inode->i_ctime = *time;
- if (flags & S_MTIME)
- inode->i_mtime = *time;
- if (!(inode->i_sb->s_flags & MS_LAZYTIME) || (flags & S_VERSION))
- iflags |= I_DIRTY_SYNC;
- __mark_inode_dirty(inode, iflags);
- return 0;
- }
- EXPORT_SYMBOL(generic_update_time);
- static int update_time(struct inode *inode, struct timespec *time, int flags)
- {
- int (*update_time)(struct inode *, struct timespec *, int);
- update_time = inode->i_op->update_time ? inode->i_op->update_time :
- generic_update_time;
- return update_time(inode, time, flags);
- }
- bool __atime_needs_update(const struct path *path, struct inode *inode,
- bool rcu)
- {
- struct vfsmount *mnt = path->mnt;
- struct timespec now;
- if (inode->i_flags & S_NOATIME)
- return false;
-
- if (HAS_UNMAPPED_ID(inode))
- return false;
- if (IS_NOATIME(inode))
- return false;
- if ((inode->i_sb->s_flags & MS_NODIRATIME) && S_ISDIR(inode->i_mode))
- return false;
- if (mnt->mnt_flags & MNT_NOATIME)
- return false;
- if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
- return false;
- now = current_time(inode);
- if (!relatime_need_update(path, inode, now, rcu))
- return false;
- if (timespec_equal(&inode->i_atime, &now))
- return false;
- return true;
- }
- void touch_atime(const struct path *path)
- {
- struct vfsmount *mnt = path->mnt;
- struct inode *inode = d_inode(path->dentry);
- struct timespec now;
- if (!__atime_needs_update(path, inode, false))
- return;
- if (!sb_start_write_trylock(inode->i_sb))
- return;
- if (__mnt_want_write(mnt) != 0)
- goto skip_update;
-
- now = current_time(inode);
- update_time(inode, &now, S_ATIME);
- __mnt_drop_write(mnt);
- skip_update:
- sb_end_write(inode->i_sb);
- }
- EXPORT_SYMBOL(touch_atime);
- int should_remove_suid(struct dentry *dentry)
- {
- umode_t mode = d_inode(dentry)->i_mode;
- int kill = 0;
-
- if (unlikely(mode & S_ISUID))
- kill = ATTR_KILL_SUID;
-
- if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
- kill |= ATTR_KILL_SGID;
- if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
- return kill;
- return 0;
- }
- EXPORT_SYMBOL(should_remove_suid);
- int dentry_needs_remove_privs(struct dentry *dentry)
- {
- struct inode *inode = d_inode(dentry);
- int mask = 0;
- int ret;
- if (IS_NOSEC(inode))
- return 0;
- mask = should_remove_suid(dentry);
- ret = security_inode_need_killpriv(dentry);
- if (ret < 0)
- return ret;
- if (ret)
- mask |= ATTR_KILL_PRIV;
- return mask;
- }
- static int __remove_privs(struct vfsmount *mnt, struct dentry *dentry, int kill)
- {
- struct iattr newattrs;
- newattrs.ia_valid = ATTR_FORCE | kill;
-
- return notify_change2(mnt, dentry, &newattrs, NULL);
- }
- int file_remove_privs(struct file *file)
- {
- struct dentry *dentry = file_dentry(file);
- struct inode *inode = file_inode(file);
- int kill;
- int error = 0;
-
- if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
- return 0;
- kill = dentry_needs_remove_privs(dentry);
- if (kill < 0)
- return kill;
- if (kill)
- error = __remove_privs(file->f_path.mnt, dentry, kill);
- if (!error)
- inode_has_no_xattr(inode);
- return error;
- }
- EXPORT_SYMBOL(file_remove_privs);
- int file_update_time(struct file *file)
- {
- struct inode *inode = file_inode(file);
- struct timespec now;
- int sync_it = 0;
- int ret;
-
- if (IS_NOCMTIME(inode))
- return 0;
- now = current_time(inode);
- if (!timespec_equal(&inode->i_mtime, &now))
- sync_it = S_MTIME;
- if (!timespec_equal(&inode->i_ctime, &now))
- sync_it |= S_CTIME;
- if (IS_I_VERSION(inode))
- sync_it |= S_VERSION;
- if (!sync_it)
- return 0;
-
- if (__mnt_want_write_file(file))
- return 0;
- ret = update_time(inode, &now, sync_it);
- __mnt_drop_write_file(file);
- return ret;
- }
- EXPORT_SYMBOL(file_update_time);
- int inode_needs_sync(struct inode *inode)
- {
- if (IS_SYNC(inode))
- return 1;
- if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
- return 1;
- return 0;
- }
- EXPORT_SYMBOL(inode_needs_sync);
- static void __wait_on_freeing_inode(struct inode *inode)
- {
- wait_queue_head_t *wq;
- DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
- wq = bit_waitqueue(&inode->i_state, __I_NEW);
- prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
- spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
- schedule();
- finish_wait(wq, &wait.wait);
- spin_lock(&inode_hash_lock);
- }
- static __initdata unsigned long ihash_entries;
- static int __init set_ihash_entries(char *str)
- {
- if (!str)
- return 0;
- ihash_entries = simple_strtoul(str, &str, 0);
- return 1;
- }
- __setup("ihash_entries=", set_ihash_entries);
- void __init inode_init_early(void)
- {
- unsigned int loop;
-
- if (hashdist)
- return;
- inode_hashtable =
- alloc_large_system_hash("Inode-cache",
- sizeof(struct hlist_head),
- ihash_entries,
- 14,
- HASH_EARLY,
- &i_hash_shift,
- &i_hash_mask,
- 0,
- 0);
- for (loop = 0; loop < (1U << i_hash_shift); loop++)
- INIT_HLIST_HEAD(&inode_hashtable[loop]);
- }
- void __init inode_init(void)
- {
- unsigned int loop;
-
- inode_cachep = kmem_cache_create("inode_cache",
- sizeof(struct inode),
- 0,
- (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
- SLAB_MEM_SPREAD|SLAB_ACCOUNT),
- init_once);
-
- if (!hashdist)
- return;
- inode_hashtable =
- alloc_large_system_hash("Inode-cache",
- sizeof(struct hlist_head),
- ihash_entries,
- 14,
- 0,
- &i_hash_shift,
- &i_hash_mask,
- 0,
- 0);
- for (loop = 0; loop < (1U << i_hash_shift); loop++)
- INIT_HLIST_HEAD(&inode_hashtable[loop]);
- }
- void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
- {
- inode->i_mode = mode;
- if (S_ISCHR(mode)) {
- inode->i_fop = &def_chr_fops;
- inode->i_rdev = rdev;
- } else if (S_ISBLK(mode)) {
- inode->i_fop = &def_blk_fops;
- inode->i_rdev = rdev;
- } else if (S_ISFIFO(mode))
- inode->i_fop = &pipefifo_fops;
- else if (S_ISSOCK(mode))
- ;
- else
- printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
- " inode %s:%lu\n", mode, inode->i_sb->s_id,
- inode->i_ino);
- }
- EXPORT_SYMBOL(init_special_inode);
- void inode_init_owner(struct inode *inode, const struct inode *dir,
- umode_t mode)
- {
- inode->i_uid = current_fsuid();
- if (dir && dir->i_mode & S_ISGID) {
- inode->i_gid = dir->i_gid;
-
- if (S_ISDIR(mode))
- mode |= S_ISGID;
- else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
- !in_group_p(inode->i_gid) &&
- !capable_wrt_inode_uidgid(dir, CAP_FSETID))
- mode &= ~S_ISGID;
- } else
- inode->i_gid = current_fsgid();
- inode->i_mode = mode;
- }
- EXPORT_SYMBOL(inode_init_owner);
- bool inode_owner_or_capable(const struct inode *inode)
- {
- struct user_namespace *ns;
- if (uid_eq(current_fsuid(), inode->i_uid))
- return true;
- ns = current_user_ns();
- if (ns_capable(ns, CAP_FOWNER) && kuid_has_mapping(ns, inode->i_uid))
- return true;
- return false;
- }
- EXPORT_SYMBOL(inode_owner_or_capable);
- static void __inode_dio_wait(struct inode *inode)
- {
- wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
- DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
- do {
- prepare_to_wait(wq, &q.wait, TASK_UNINTERRUPTIBLE);
- if (atomic_read(&inode->i_dio_count))
- schedule();
- } while (atomic_read(&inode->i_dio_count));
- finish_wait(wq, &q.wait);
- }
- void inode_dio_wait(struct inode *inode)
- {
- if (atomic_read(&inode->i_dio_count))
- __inode_dio_wait(inode);
- }
- EXPORT_SYMBOL(inode_dio_wait);
- void inode_set_flags(struct inode *inode, unsigned int flags,
- unsigned int mask)
- {
- unsigned int old_flags, new_flags;
- WARN_ON_ONCE(flags & ~mask);
- do {
- old_flags = ACCESS_ONCE(inode->i_flags);
- new_flags = (old_flags & ~mask) | flags;
- } while (unlikely(cmpxchg(&inode->i_flags, old_flags,
- new_flags) != old_flags));
- }
- EXPORT_SYMBOL(inode_set_flags);
- void inode_nohighmem(struct inode *inode)
- {
- mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
- }
- EXPORT_SYMBOL(inode_nohighmem);
- struct timespec current_time(struct inode *inode)
- {
- struct timespec now = current_kernel_time();
- if (unlikely(!inode->i_sb)) {
- WARN(1, "current_time() called with uninitialized super_block in the inode");
- return now;
- }
- return timespec_trunc(now, inode->i_sb->s_time_gran);
- }
- EXPORT_SYMBOL(current_time);
|