1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109 |
- #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
- #include <linux/capability.h>
- #include <linux/errno.h>
- #include <linux/errqueue.h>
- #include <linux/types.h>
- #include <linux/socket.h>
- #include <linux/in.h>
- #include <linux/kernel.h>
- #include <linux/module.h>
- #include <linux/proc_fs.h>
- #include <linux/seq_file.h>
- #include <linux/sched.h>
- #include <linux/timer.h>
- #include <linux/string.h>
- #include <linux/sockios.h>
- #include <linux/net.h>
- #include <linux/mm.h>
- #include <linux/slab.h>
- #include <linux/interrupt.h>
- #include <linux/poll.h>
- #include <linux/tcp.h>
- #include <linux/init.h>
- #include <linux/highmem.h>
- #include <linux/user_namespace.h>
- #include <linux/static_key.h>
- #include <linux/memcontrol.h>
- #include <linux/prefetch.h>
- #include <asm/uaccess.h>
- #include <linux/netdevice.h>
- #include <net/protocol.h>
- #include <linux/skbuff.h>
- #include <net/net_namespace.h>
- #include <net/request_sock.h>
- #include <net/sock.h>
- #include <linux/net_tstamp.h>
- #include <net/xfrm.h>
- #include <linux/ipsec.h>
- #include <net/cls_cgroup.h>
- #include <net/netprio_cgroup.h>
- #include <linux/sock_diag.h>
- #include <linux/filter.h>
- #include <net/sock_reuseport.h>
- #include <trace/events/sock.h>
- #include <net/tcp.h>
- #include <net/busy_poll.h>
- static DEFINE_MUTEX(proto_list_mutex);
- static LIST_HEAD(proto_list);
- bool sk_ns_capable(const struct sock *sk,
- struct user_namespace *user_ns, int cap)
- {
- return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
- ns_capable(user_ns, cap);
- }
- EXPORT_SYMBOL(sk_ns_capable);
- bool sk_capable(const struct sock *sk, int cap)
- {
- return sk_ns_capable(sk, &init_user_ns, cap);
- }
- EXPORT_SYMBOL(sk_capable);
- bool sk_net_capable(const struct sock *sk, int cap)
- {
- return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
- }
- EXPORT_SYMBOL(sk_net_capable);
- static struct lock_class_key af_family_keys[AF_MAX];
- static struct lock_class_key af_family_slock_keys[AF_MAX];
- static const char *const af_family_key_strings[AF_MAX+1] = {
- "sk_lock-AF_UNSPEC", "sk_lock-AF_UNIX" , "sk_lock-AF_INET" ,
- "sk_lock-AF_AX25" , "sk_lock-AF_IPX" , "sk_lock-AF_APPLETALK",
- "sk_lock-AF_NETROM", "sk_lock-AF_BRIDGE" , "sk_lock-AF_ATMPVC" ,
- "sk_lock-AF_X25" , "sk_lock-AF_INET6" , "sk_lock-AF_ROSE" ,
- "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" ,
- "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" ,
- "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" ,
- "sk_lock-AF_RDS" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" ,
- "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" ,
- "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" ,
- "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" ,
- "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" ,
- "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" ,
- "sk_lock-AF_NFC" , "sk_lock-AF_VSOCK" , "sk_lock-AF_KCM" ,
- "sk_lock-AF_QIPCRTR", "sk_lock-AF_MAX"
- };
- static const char *const af_family_slock_key_strings[AF_MAX+1] = {
- "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" ,
- "slock-AF_AX25" , "slock-AF_IPX" , "slock-AF_APPLETALK",
- "slock-AF_NETROM", "slock-AF_BRIDGE" , "slock-AF_ATMPVC" ,
- "slock-AF_X25" , "slock-AF_INET6" , "slock-AF_ROSE" ,
- "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" ,
- "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" ,
- "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" ,
- "slock-AF_RDS" , "slock-AF_SNA" , "slock-AF_IRDA" ,
- "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" ,
- "slock-27" , "slock-28" , "slock-AF_CAN" ,
- "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" ,
- "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" ,
- "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" ,
- "slock-AF_NFC" , "slock-AF_VSOCK" ,"slock-AF_KCM" ,
- "slock-AF_QIPCRTR", "slock-AF_MAX"
- };
- static const char *const af_family_clock_key_strings[AF_MAX+1] = {
- "clock-AF_UNSPEC", "clock-AF_UNIX" , "clock-AF_INET" ,
- "clock-AF_AX25" , "clock-AF_IPX" , "clock-AF_APPLETALK",
- "clock-AF_NETROM", "clock-AF_BRIDGE" , "clock-AF_ATMPVC" ,
- "clock-AF_X25" , "clock-AF_INET6" , "clock-AF_ROSE" ,
- "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" ,
- "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" ,
- "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" ,
- "clock-AF_RDS" , "clock-AF_SNA" , "clock-AF_IRDA" ,
- "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" ,
- "clock-27" , "clock-28" , "clock-AF_CAN" ,
- "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" ,
- "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" ,
- "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" ,
- "clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_KCM" ,
- "clock-AF_QIPCRTR", "clock-AF_MAX"
- };
- static struct lock_class_key af_callback_keys[AF_MAX];
- #define _SK_MEM_PACKETS 256
- #define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
- #define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
- #define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
- __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
- EXPORT_SYMBOL(sysctl_wmem_max);
- __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
- EXPORT_SYMBOL(sysctl_rmem_max);
- __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
- __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
- int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
- EXPORT_SYMBOL(sysctl_optmem_max);
- int sysctl_tstamp_allow_data __read_mostly = 1;
- struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
- EXPORT_SYMBOL_GPL(memalloc_socks);
- void sk_set_memalloc(struct sock *sk)
- {
- sock_set_flag(sk, SOCK_MEMALLOC);
- sk->sk_allocation |= __GFP_MEMALLOC;
- static_key_slow_inc(&memalloc_socks);
- }
- EXPORT_SYMBOL_GPL(sk_set_memalloc);
- void sk_clear_memalloc(struct sock *sk)
- {
- sock_reset_flag(sk, SOCK_MEMALLOC);
- sk->sk_allocation &= ~__GFP_MEMALLOC;
- static_key_slow_dec(&memalloc_socks);
-
- sk_mem_reclaim(sk);
- }
- EXPORT_SYMBOL_GPL(sk_clear_memalloc);
- int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
- {
- int ret;
- unsigned long pflags = current->flags;
-
- BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
- current->flags |= PF_MEMALLOC;
- ret = sk->sk_backlog_rcv(sk, skb);
- tsk_restore_flags(current, pflags, PF_MEMALLOC);
- return ret;
- }
- EXPORT_SYMBOL(__sk_backlog_rcv);
- static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
- {
- struct timeval tv;
- if (optlen < sizeof(tv))
- return -EINVAL;
- if (copy_from_user(&tv, optval, sizeof(tv)))
- return -EFAULT;
- if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
- return -EDOM;
- if (tv.tv_sec < 0) {
- static int warned __read_mostly;
- *timeo_p = 0;
- if (warned < 10 && net_ratelimit()) {
- warned++;
- pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
- __func__, current->comm, task_pid_nr(current));
- }
- return 0;
- }
- *timeo_p = MAX_SCHEDULE_TIMEOUT;
- if (tv.tv_sec == 0 && tv.tv_usec == 0)
- return 0;
- if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
- *timeo_p = tv.tv_sec*HZ + (tv.tv_usec+(1000000/HZ-1))/(1000000/HZ);
- return 0;
- }
- static void sock_warn_obsolete_bsdism(const char *name)
- {
- static int warned;
- static char warncomm[TASK_COMM_LEN];
- if (strcmp(warncomm, current->comm) && warned < 5) {
- strcpy(warncomm, current->comm);
- pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
- warncomm, name);
- warned++;
- }
- }
- static bool sock_needs_netstamp(const struct sock *sk)
- {
- switch (sk->sk_family) {
- case AF_UNSPEC:
- case AF_UNIX:
- return false;
- default:
- return true;
- }
- }
- static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
- {
- if (sk->sk_flags & flags) {
- sk->sk_flags &= ~flags;
- if (sock_needs_netstamp(sk) &&
- !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
- net_disable_timestamp();
- }
- }
- int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
- {
- unsigned long flags;
- struct sk_buff_head *list = &sk->sk_receive_queue;
- if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
- atomic_inc(&sk->sk_drops);
- trace_sock_rcvqueue_full(sk, skb);
- return -ENOMEM;
- }
- if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
- atomic_inc(&sk->sk_drops);
- return -ENOBUFS;
- }
- skb->dev = NULL;
- skb_set_owner_r(skb, sk);
-
- skb_dst_force(skb);
- spin_lock_irqsave(&list->lock, flags);
- sock_skb_set_dropcount(sk, skb);
- __skb_queue_tail(list, skb);
- spin_unlock_irqrestore(&list->lock, flags);
- if (!sock_flag(sk, SOCK_DEAD))
- sk->sk_data_ready(sk);
- return 0;
- }
- EXPORT_SYMBOL(__sock_queue_rcv_skb);
- int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
- {
- int err;
- err = sk_filter(sk, skb);
- if (err)
- return err;
- return __sock_queue_rcv_skb(sk, skb);
- }
- EXPORT_SYMBOL(sock_queue_rcv_skb);
- int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
- const int nested, unsigned int trim_cap, bool refcounted)
- {
- int rc = NET_RX_SUCCESS;
- if (sk_filter_trim_cap(sk, skb, trim_cap))
- goto discard_and_relse;
- skb->dev = NULL;
- if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
- atomic_inc(&sk->sk_drops);
- goto discard_and_relse;
- }
- if (nested)
- bh_lock_sock_nested(sk);
- else
- bh_lock_sock(sk);
- if (!sock_owned_by_user(sk)) {
-
- mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
- rc = sk_backlog_rcv(sk, skb);
- mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
- } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
- bh_unlock_sock(sk);
- atomic_inc(&sk->sk_drops);
- goto discard_and_relse;
- }
- bh_unlock_sock(sk);
- out:
- if (refcounted)
- sock_put(sk);
- return rc;
- discard_and_relse:
- kfree_skb(skb);
- goto out;
- }
- EXPORT_SYMBOL(__sk_receive_skb);
- struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
- {
- struct dst_entry *dst = __sk_dst_get(sk);
- if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
- sk_tx_queue_clear(sk);
- RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
- dst_release(dst);
- return NULL;
- }
- return dst;
- }
- EXPORT_SYMBOL(__sk_dst_check);
- struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
- {
- struct dst_entry *dst = sk_dst_get(sk);
- if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
- sk_dst_reset(sk);
- dst_release(dst);
- return NULL;
- }
- return dst;
- }
- EXPORT_SYMBOL(sk_dst_check);
- static int sock_setbindtodevice(struct sock *sk, char __user *optval,
- int optlen)
- {
- int ret = -ENOPROTOOPT;
- #ifdef CONFIG_NETDEVICES
- struct net *net = sock_net(sk);
- char devname[IFNAMSIZ];
- int index;
-
- ret = -EPERM;
- if (!ns_capable(net->user_ns, CAP_NET_RAW))
- goto out;
- ret = -EINVAL;
- if (optlen < 0)
- goto out;
-
- if (optlen > IFNAMSIZ - 1)
- optlen = IFNAMSIZ - 1;
- memset(devname, 0, sizeof(devname));
- ret = -EFAULT;
- if (copy_from_user(devname, optval, optlen))
- goto out;
- index = 0;
- if (devname[0] != '\0') {
- struct net_device *dev;
- rcu_read_lock();
- dev = dev_get_by_name_rcu(net, devname);
- if (dev)
- index = dev->ifindex;
- rcu_read_unlock();
- ret = -ENODEV;
- if (!dev)
- goto out;
- }
- lock_sock(sk);
- sk->sk_bound_dev_if = index;
- sk_dst_reset(sk);
- release_sock(sk);
- ret = 0;
- out:
- #endif
- return ret;
- }
- static int sock_getbindtodevice(struct sock *sk, char __user *optval,
- int __user *optlen, int len)
- {
- int ret = -ENOPROTOOPT;
- #ifdef CONFIG_NETDEVICES
- struct net *net = sock_net(sk);
- char devname[IFNAMSIZ];
- if (sk->sk_bound_dev_if == 0) {
- len = 0;
- goto zero;
- }
- ret = -EINVAL;
- if (len < IFNAMSIZ)
- goto out;
- ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
- if (ret)
- goto out;
- len = strlen(devname) + 1;
- ret = -EFAULT;
- if (copy_to_user(optval, devname, len))
- goto out;
- zero:
- ret = -EFAULT;
- if (put_user(len, optlen))
- goto out;
- ret = 0;
- out:
- #endif
- return ret;
- }
- static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
- {
- if (valbool)
- sock_set_flag(sk, bit);
- else
- sock_reset_flag(sk, bit);
- }
- bool sk_mc_loop(struct sock *sk)
- {
- if (dev_recursion_level())
- return false;
- if (!sk)
- return true;
- switch (sk->sk_family) {
- case AF_INET:
- return inet_sk(sk)->mc_loop;
- #if IS_ENABLED(CONFIG_IPV6)
- case AF_INET6:
- return inet6_sk(sk)->mc_loop;
- #endif
- }
- WARN_ON(1);
- return true;
- }
- EXPORT_SYMBOL(sk_mc_loop);
- int sock_setsockopt(struct socket *sock, int level, int optname,
- char __user *optval, unsigned int optlen)
- {
- struct sock *sk = sock->sk;
- int val;
- int valbool;
- struct linger ling;
- int ret = 0;
-
- if (optname == SO_BINDTODEVICE)
- return sock_setbindtodevice(sk, optval, optlen);
- if (optlen < sizeof(int))
- return -EINVAL;
- if (get_user(val, (int __user *)optval))
- return -EFAULT;
- valbool = val ? 1 : 0;
- lock_sock(sk);
- switch (optname) {
- case SO_DEBUG:
- if (val && !capable(CAP_NET_ADMIN))
- ret = -EACCES;
- else
- sock_valbool_flag(sk, SOCK_DBG, valbool);
- break;
- case SO_REUSEADDR:
- sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
- break;
- case SO_REUSEPORT:
- sk->sk_reuseport = valbool;
- break;
- case SO_TYPE:
- case SO_PROTOCOL:
- case SO_DOMAIN:
- case SO_ERROR:
- ret = -ENOPROTOOPT;
- break;
- case SO_DONTROUTE:
- sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
- sk_dst_reset(sk);
- break;
- case SO_BROADCAST:
- sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
- break;
- case SO_SNDBUF:
-
- val = min_t(u32, val, sysctl_wmem_max);
- set_sndbuf:
- sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
- sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
-
- sk->sk_write_space(sk);
- break;
- case SO_SNDBUFFORCE:
- if (!capable(CAP_NET_ADMIN)) {
- ret = -EPERM;
- break;
- }
- goto set_sndbuf;
- case SO_RCVBUF:
-
- val = min_t(u32, val, sysctl_rmem_max);
- set_rcvbuf:
- sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
-
- sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
- break;
- case SO_RCVBUFFORCE:
- if (!capable(CAP_NET_ADMIN)) {
- ret = -EPERM;
- break;
- }
- goto set_rcvbuf;
- case SO_KEEPALIVE:
- #ifdef CONFIG_INET
- if (sk->sk_protocol == IPPROTO_TCP &&
- sk->sk_type == SOCK_STREAM)
- tcp_set_keepalive(sk, valbool);
- #endif
- sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
- break;
- case SO_OOBINLINE:
- sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
- break;
- case SO_NO_CHECK:
- sk->sk_no_check_tx = valbool;
- break;
- case SO_PRIORITY:
- if ((val >= 0 && val <= 6) ||
- ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
- sk->sk_priority = val;
- else
- ret = -EPERM;
- break;
- case SO_LINGER:
- if (optlen < sizeof(ling)) {
- ret = -EINVAL;
- break;
- }
- if (copy_from_user(&ling, optval, sizeof(ling))) {
- ret = -EFAULT;
- break;
- }
- if (!ling.l_onoff)
- sock_reset_flag(sk, SOCK_LINGER);
- else {
- #if (BITS_PER_LONG == 32)
- if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
- sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
- else
- #endif
- sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
- sock_set_flag(sk, SOCK_LINGER);
- }
- break;
- case SO_BSDCOMPAT:
- sock_warn_obsolete_bsdism("setsockopt");
- break;
- case SO_PASSCRED:
- if (valbool)
- set_bit(SOCK_PASSCRED, &sock->flags);
- else
- clear_bit(SOCK_PASSCRED, &sock->flags);
- break;
- case SO_TIMESTAMP:
- case SO_TIMESTAMPNS:
- if (valbool) {
- if (optname == SO_TIMESTAMP)
- sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
- else
- sock_set_flag(sk, SOCK_RCVTSTAMPNS);
- sock_set_flag(sk, SOCK_RCVTSTAMP);
- sock_enable_timestamp(sk, SOCK_TIMESTAMP);
- } else {
- sock_reset_flag(sk, SOCK_RCVTSTAMP);
- sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
- }
- break;
- case SO_TIMESTAMPING:
- if (val & ~SOF_TIMESTAMPING_MASK) {
- ret = -EINVAL;
- break;
- }
- if (val & SOF_TIMESTAMPING_OPT_ID &&
- !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
- if (sk->sk_protocol == IPPROTO_TCP &&
- sk->sk_type == SOCK_STREAM) {
- if ((1 << sk->sk_state) &
- (TCPF_CLOSE | TCPF_LISTEN)) {
- ret = -EINVAL;
- break;
- }
- sk->sk_tskey = tcp_sk(sk)->snd_una;
- } else {
- sk->sk_tskey = 0;
- }
- }
- sk->sk_tsflags = val;
- if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
- sock_enable_timestamp(sk,
- SOCK_TIMESTAMPING_RX_SOFTWARE);
- else
- sock_disable_timestamp(sk,
- (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
- break;
- case SO_RCVLOWAT:
- if (val < 0)
- val = INT_MAX;
- sk->sk_rcvlowat = val ? : 1;
- break;
- case SO_RCVTIMEO:
- ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
- break;
- case SO_SNDTIMEO:
- ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
- break;
- case SO_ATTACH_FILTER:
- ret = -EINVAL;
- if (optlen == sizeof(struct sock_fprog)) {
- struct sock_fprog fprog;
- ret = -EFAULT;
- if (copy_from_user(&fprog, optval, sizeof(fprog)))
- break;
- ret = sk_attach_filter(&fprog, sk);
- }
- break;
- case SO_ATTACH_BPF:
- ret = -EINVAL;
- if (optlen == sizeof(u32)) {
- u32 ufd;
- ret = -EFAULT;
- if (copy_from_user(&ufd, optval, sizeof(ufd)))
- break;
- ret = sk_attach_bpf(ufd, sk);
- }
- break;
- case SO_ATTACH_REUSEPORT_CBPF:
- ret = -EINVAL;
- if (optlen == sizeof(struct sock_fprog)) {
- struct sock_fprog fprog;
- ret = -EFAULT;
- if (copy_from_user(&fprog, optval, sizeof(fprog)))
- break;
- ret = sk_reuseport_attach_filter(&fprog, sk);
- }
- break;
- case SO_ATTACH_REUSEPORT_EBPF:
- ret = -EINVAL;
- if (optlen == sizeof(u32)) {
- u32 ufd;
- ret = -EFAULT;
- if (copy_from_user(&ufd, optval, sizeof(ufd)))
- break;
- ret = sk_reuseport_attach_bpf(ufd, sk);
- }
- break;
- case SO_DETACH_FILTER:
- ret = sk_detach_filter(sk);
- break;
- case SO_LOCK_FILTER:
- if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
- ret = -EPERM;
- else
- sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
- break;
- case SO_PASSSEC:
- if (valbool)
- set_bit(SOCK_PASSSEC, &sock->flags);
- else
- clear_bit(SOCK_PASSSEC, &sock->flags);
- break;
- case SO_MARK:
- if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
- ret = -EPERM;
- else
- sk->sk_mark = val;
- break;
- case SO_RXQ_OVFL:
- sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
- break;
- case SO_WIFI_STATUS:
- sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
- break;
- case SO_PEEK_OFF:
- if (sock->ops->set_peek_off)
- ret = sock->ops->set_peek_off(sk, val);
- else
- ret = -EOPNOTSUPP;
- break;
- case SO_NOFCS:
- sock_valbool_flag(sk, SOCK_NOFCS, valbool);
- break;
- case SO_SELECT_ERR_QUEUE:
- sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
- break;
- #ifdef CONFIG_NET_RX_BUSY_POLL
- case SO_BUSY_POLL:
-
- if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
- ret = -EPERM;
- else {
- if (val < 0)
- ret = -EINVAL;
- else
- sk->sk_ll_usec = val;
- }
- break;
- #endif
- case SO_MAX_PACING_RATE:
- sk->sk_max_pacing_rate = val;
- sk->sk_pacing_rate = min(sk->sk_pacing_rate,
- sk->sk_max_pacing_rate);
- break;
- case SO_INCOMING_CPU:
- sk->sk_incoming_cpu = val;
- break;
- case SO_CNX_ADVICE:
- if (val == 1)
- dst_negative_advice(sk);
- break;
- default:
- ret = -ENOPROTOOPT;
- break;
- }
- release_sock(sk);
- return ret;
- }
- EXPORT_SYMBOL(sock_setsockopt);
- static void cred_to_ucred(struct pid *pid, const struct cred *cred,
- struct ucred *ucred)
- {
- ucred->pid = pid_vnr(pid);
- ucred->uid = ucred->gid = -1;
- if (cred) {
- struct user_namespace *current_ns = current_user_ns();
- ucred->uid = from_kuid_munged(current_ns, cred->euid);
- ucred->gid = from_kgid_munged(current_ns, cred->egid);
- }
- }
- int sock_getsockopt(struct socket *sock, int level, int optname,
- char __user *optval, int __user *optlen)
- {
- struct sock *sk = sock->sk;
- union {
- int val;
- u64 val64;
- struct linger ling;
- struct timeval tm;
- } v;
- int lv = sizeof(int);
- int len;
- if (get_user(len, optlen))
- return -EFAULT;
- if (len < 0)
- return -EINVAL;
- memset(&v, 0, sizeof(v));
- switch (optname) {
- case SO_DEBUG:
- v.val = sock_flag(sk, SOCK_DBG);
- break;
- case SO_DONTROUTE:
- v.val = sock_flag(sk, SOCK_LOCALROUTE);
- break;
- case SO_BROADCAST:
- v.val = sock_flag(sk, SOCK_BROADCAST);
- break;
- case SO_SNDBUF:
- v.val = sk->sk_sndbuf;
- break;
- case SO_RCVBUF:
- v.val = sk->sk_rcvbuf;
- break;
- case SO_REUSEADDR:
- v.val = sk->sk_reuse;
- break;
- case SO_REUSEPORT:
- v.val = sk->sk_reuseport;
- break;
- case SO_KEEPALIVE:
- v.val = sock_flag(sk, SOCK_KEEPOPEN);
- break;
- case SO_TYPE:
- v.val = sk->sk_type;
- break;
- case SO_PROTOCOL:
- v.val = sk->sk_protocol;
- break;
- case SO_DOMAIN:
- v.val = sk->sk_family;
- break;
- case SO_ERROR:
- v.val = -sock_error(sk);
- if (v.val == 0)
- v.val = xchg(&sk->sk_err_soft, 0);
- break;
- case SO_OOBINLINE:
- v.val = sock_flag(sk, SOCK_URGINLINE);
- break;
- case SO_NO_CHECK:
- v.val = sk->sk_no_check_tx;
- break;
- case SO_PRIORITY:
- v.val = sk->sk_priority;
- break;
- case SO_LINGER:
- lv = sizeof(v.ling);
- v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
- v.ling.l_linger = sk->sk_lingertime / HZ;
- break;
- case SO_BSDCOMPAT:
- sock_warn_obsolete_bsdism("getsockopt");
- break;
- case SO_TIMESTAMP:
- v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
- !sock_flag(sk, SOCK_RCVTSTAMPNS);
- break;
- case SO_TIMESTAMPNS:
- v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
- break;
- case SO_TIMESTAMPING:
- v.val = sk->sk_tsflags;
- break;
- case SO_RCVTIMEO:
- lv = sizeof(struct timeval);
- if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
- v.tm.tv_sec = 0;
- v.tm.tv_usec = 0;
- } else {
- v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
- v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * 1000000) / HZ;
- }
- break;
- case SO_SNDTIMEO:
- lv = sizeof(struct timeval);
- if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
- v.tm.tv_sec = 0;
- v.tm.tv_usec = 0;
- } else {
- v.tm.tv_sec = sk->sk_sndtimeo / HZ;
- v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * 1000000) / HZ;
- }
- break;
- case SO_RCVLOWAT:
- v.val = sk->sk_rcvlowat;
- break;
- case SO_SNDLOWAT:
- v.val = 1;
- break;
- case SO_PASSCRED:
- v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
- break;
- case SO_PEERCRED:
- {
- struct ucred peercred;
- if (len > sizeof(peercred))
- len = sizeof(peercred);
- cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
- if (copy_to_user(optval, &peercred, len))
- return -EFAULT;
- goto lenout;
- }
- case SO_PEERNAME:
- {
- char address[128];
- if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
- return -ENOTCONN;
- if (lv < len)
- return -EINVAL;
- if (copy_to_user(optval, address, len))
- return -EFAULT;
- goto lenout;
- }
-
- case SO_ACCEPTCONN:
- v.val = sk->sk_state == TCP_LISTEN;
- break;
- case SO_PASSSEC:
- v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
- break;
- case SO_PEERSEC:
- return security_socket_getpeersec_stream(sock, optval, optlen, len);
- case SO_MARK:
- v.val = sk->sk_mark;
- break;
- case SO_RXQ_OVFL:
- v.val = sock_flag(sk, SOCK_RXQ_OVFL);
- break;
- case SO_WIFI_STATUS:
- v.val = sock_flag(sk, SOCK_WIFI_STATUS);
- break;
- case SO_PEEK_OFF:
- if (!sock->ops->set_peek_off)
- return -EOPNOTSUPP;
- v.val = sk->sk_peek_off;
- break;
- case SO_NOFCS:
- v.val = sock_flag(sk, SOCK_NOFCS);
- break;
- case SO_BINDTODEVICE:
- return sock_getbindtodevice(sk, optval, optlen, len);
- case SO_GET_FILTER:
- len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
- if (len < 0)
- return len;
- goto lenout;
- case SO_LOCK_FILTER:
- v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
- break;
- case SO_BPF_EXTENSIONS:
- v.val = bpf_tell_extensions();
- break;
- case SO_SELECT_ERR_QUEUE:
- v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
- break;
- #ifdef CONFIG_NET_RX_BUSY_POLL
- case SO_BUSY_POLL:
- v.val = sk->sk_ll_usec;
- break;
- #endif
- case SO_MAX_PACING_RATE:
- v.val = sk->sk_max_pacing_rate;
- break;
- case SO_INCOMING_CPU:
- v.val = sk->sk_incoming_cpu;
- break;
- case SO_COOKIE:
- lv = sizeof(u64);
- if (len < lv)
- return -EINVAL;
- v.val64 = sock_gen_cookie(sk);
- break;
- default:
-
- return -ENOPROTOOPT;
- }
- if (len > lv)
- len = lv;
- if (copy_to_user(optval, &v, len))
- return -EFAULT;
- lenout:
- if (put_user(len, optlen))
- return -EFAULT;
- return 0;
- }
- static inline void sock_lock_init(struct sock *sk)
- {
- sock_lock_init_class_and_name(sk,
- af_family_slock_key_strings[sk->sk_family],
- af_family_slock_keys + sk->sk_family,
- af_family_key_strings[sk->sk_family],
- af_family_keys + sk->sk_family);
- }
- static void sock_copy(struct sock *nsk, const struct sock *osk)
- {
- #ifdef CONFIG_SECURITY_NETWORK
- void *sptr = nsk->sk_security;
- #endif
- memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
- memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
- osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
- #ifdef CONFIG_SECURITY_NETWORK
- nsk->sk_security = sptr;
- security_sk_clone(osk, nsk);
- #endif
- }
- static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
- int family)
- {
- struct sock *sk;
- struct kmem_cache *slab;
- slab = prot->slab;
- if (slab != NULL) {
- sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
- if (!sk)
- return sk;
- if (priority & __GFP_ZERO)
- sk_prot_clear_nulls(sk, prot->obj_size);
- } else
- sk = kmalloc(prot->obj_size, priority);
- if (sk != NULL) {
- kmemcheck_annotate_bitfield(sk, flags);
- if (security_sk_alloc(sk, family, priority))
- goto out_free;
- if (!try_module_get(prot->owner))
- goto out_free_sec;
- sk_tx_queue_clear(sk);
- }
- return sk;
- out_free_sec:
- security_sk_free(sk);
- out_free:
- if (slab != NULL)
- kmem_cache_free(slab, sk);
- else
- kfree(sk);
- return NULL;
- }
- static void sk_prot_free(struct proto *prot, struct sock *sk)
- {
- struct kmem_cache *slab;
- struct module *owner;
- owner = prot->owner;
- slab = prot->slab;
- cgroup_sk_free(&sk->sk_cgrp_data);
- mem_cgroup_sk_free(sk);
- security_sk_free(sk);
- if (slab != NULL)
- kmem_cache_free(slab, sk);
- else
- kfree(sk);
- module_put(owner);
- }
- struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
- struct proto *prot, int kern)
- {
- struct sock *sk;
- sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
- if (sk) {
- sk->sk_family = family;
-
- sk->sk_prot = sk->sk_prot_creator = prot;
- sock_lock_init(sk);
- sk->sk_net_refcnt = kern ? 0 : 1;
- if (likely(sk->sk_net_refcnt))
- get_net(net);
- sock_net_set(sk, net);
- atomic_set(&sk->sk_wmem_alloc, 1);
- mem_cgroup_sk_alloc(sk);
- cgroup_sk_alloc(&sk->sk_cgrp_data);
- sock_update_classid(&sk->sk_cgrp_data);
- sock_update_netprioidx(&sk->sk_cgrp_data);
- }
- return sk;
- }
- EXPORT_SYMBOL(sk_alloc);
- static void __sk_destruct(struct rcu_head *head)
- {
- struct sock *sk = container_of(head, struct sock, sk_rcu);
- struct sk_filter *filter;
- if (sk->sk_destruct)
- sk->sk_destruct(sk);
- filter = rcu_dereference_check(sk->sk_filter,
- atomic_read(&sk->sk_wmem_alloc) == 0);
- if (filter) {
- sk_filter_uncharge(sk, filter);
- RCU_INIT_POINTER(sk->sk_filter, NULL);
- }
- sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
- if (atomic_read(&sk->sk_omem_alloc))
- pr_debug("%s: optmem leakage (%d bytes) detected\n",
- __func__, atomic_read(&sk->sk_omem_alloc));
- if (sk->sk_frag.page) {
- put_page(sk->sk_frag.page);
- sk->sk_frag.page = NULL;
- }
- if (sk->sk_peer_cred)
- put_cred(sk->sk_peer_cred);
- put_pid(sk->sk_peer_pid);
- if (likely(sk->sk_net_refcnt))
- put_net(sock_net(sk));
- sk_prot_free(sk->sk_prot_creator, sk);
- }
- void sk_destruct(struct sock *sk)
- {
- bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
- if (rcu_access_pointer(sk->sk_reuseport_cb)) {
- reuseport_detach_sock(sk);
- use_call_rcu = true;
- }
- if (use_call_rcu)
- call_rcu(&sk->sk_rcu, __sk_destruct);
- else
- __sk_destruct(&sk->sk_rcu);
- }
- static void __sk_free(struct sock *sk)
- {
- if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
- sock_diag_broadcast_destroy(sk);
- else
- sk_destruct(sk);
- }
- void sk_free(struct sock *sk)
- {
-
- if (atomic_dec_and_test(&sk->sk_wmem_alloc))
- __sk_free(sk);
- }
- EXPORT_SYMBOL(sk_free);
- struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
- {
- struct sock *newsk;
- bool is_charged = true;
- newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
- if (newsk != NULL) {
- struct sk_filter *filter;
- sock_copy(newsk, sk);
- newsk->sk_prot_creator = sk->sk_prot;
-
- if (likely(newsk->sk_net_refcnt))
- get_net(sock_net(newsk));
- sk_node_init(&newsk->sk_node);
- sock_lock_init(newsk);
- bh_lock_sock(newsk);
- newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
- newsk->sk_backlog.len = 0;
- atomic_set(&newsk->sk_rmem_alloc, 0);
-
- atomic_set(&newsk->sk_wmem_alloc, 1);
- atomic_set(&newsk->sk_omem_alloc, 0);
- skb_queue_head_init(&newsk->sk_receive_queue);
- skb_queue_head_init(&newsk->sk_write_queue);
- rwlock_init(&newsk->sk_callback_lock);
- lockdep_set_class_and_name(&newsk->sk_callback_lock,
- af_callback_keys + newsk->sk_family,
- af_family_clock_key_strings[newsk->sk_family]);
- newsk->sk_dst_cache = NULL;
- newsk->sk_wmem_queued = 0;
- newsk->sk_forward_alloc = 0;
- atomic_set(&newsk->sk_drops, 0);
- newsk->sk_send_head = NULL;
- newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
- sock_reset_flag(newsk, SOCK_DONE);
- cgroup_sk_alloc(&newsk->sk_cgrp_data);
- skb_queue_head_init(&newsk->sk_error_queue);
- filter = rcu_dereference_protected(newsk->sk_filter, 1);
- if (filter != NULL)
-
- is_charged = sk_filter_charge(newsk, filter);
- if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
-
- if (!is_charged)
- RCU_INIT_POINTER(newsk->sk_filter, NULL);
-
- newsk->sk_destruct = NULL;
- bh_unlock_sock(newsk);
- sk_free(newsk);
- newsk = NULL;
- goto out;
- }
- RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
- newsk->sk_err = 0;
- newsk->sk_err_soft = 0;
- newsk->sk_priority = 0;
- newsk->sk_incoming_cpu = raw_smp_processor_id();
- atomic64_set(&newsk->sk_cookie, 0);
- mem_cgroup_sk_alloc(newsk);
-
- smp_wmb();
- atomic_set(&newsk->sk_refcnt, 2);
-
- sk_refcnt_debug_inc(newsk);
- sk_set_socket(newsk, NULL);
- newsk->sk_wq = NULL;
- if (newsk->sk_prot->sockets_allocated)
- sk_sockets_allocated_inc(newsk);
- if (sock_needs_netstamp(sk) &&
- newsk->sk_flags & SK_FLAGS_TIMESTAMP)
- net_enable_timestamp();
- }
- out:
- return newsk;
- }
- EXPORT_SYMBOL_GPL(sk_clone_lock);
- void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
- {
- u32 max_segs = 1;
- sk_dst_set(sk, dst);
- sk->sk_route_caps = dst->dev->features;
- if (sk->sk_route_caps & NETIF_F_GSO)
- sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
- sk->sk_route_caps &= ~sk->sk_route_nocaps;
- if (sk_can_gso(sk)) {
- if (dst->header_len) {
- sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
- } else {
- sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
- sk->sk_gso_max_size = dst->dev->gso_max_size;
- max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
- }
- }
- sk->sk_gso_max_segs = max_segs;
- }
- EXPORT_SYMBOL_GPL(sk_setup_caps);
- void sock_wfree(struct sk_buff *skb)
- {
- struct sock *sk = skb->sk;
- unsigned int len = skb->truesize;
- if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
-
- atomic_sub(len - 1, &sk->sk_wmem_alloc);
- sk->sk_write_space(sk);
- len = 1;
- }
-
- if (atomic_sub_and_test(len, &sk->sk_wmem_alloc))
- __sk_free(sk);
- }
- EXPORT_SYMBOL(sock_wfree);
- void __sock_wfree(struct sk_buff *skb)
- {
- struct sock *sk = skb->sk;
- if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
- __sk_free(sk);
- }
- void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
- {
- skb_orphan(skb);
- skb->sk = sk;
- #ifdef CONFIG_INET
- if (unlikely(!sk_fullsock(sk))) {
- skb->destructor = sock_edemux;
- sock_hold(sk);
- return;
- }
- #endif
- skb->destructor = sock_wfree;
- skb_set_hash_from_sk(skb, sk);
-
- atomic_add(skb->truesize, &sk->sk_wmem_alloc);
- }
- EXPORT_SYMBOL(skb_set_owner_w);
- void skb_orphan_partial(struct sk_buff *skb)
- {
- if (skb_is_tcp_pure_ack(skb))
- return;
- if (skb->destructor == sock_wfree
- #ifdef CONFIG_INET
- || skb->destructor == tcp_wfree
- #endif
- ) {
- struct sock *sk = skb->sk;
- if (atomic_inc_not_zero(&sk->sk_refcnt)) {
- atomic_sub(skb->truesize, &sk->sk_wmem_alloc);
- skb->destructor = sock_efree;
- }
- } else {
- skb_orphan(skb);
- }
- }
- EXPORT_SYMBOL(skb_orphan_partial);
- void sock_rfree(struct sk_buff *skb)
- {
- struct sock *sk = skb->sk;
- unsigned int len = skb->truesize;
- atomic_sub(len, &sk->sk_rmem_alloc);
- sk_mem_uncharge(sk, len);
- }
- EXPORT_SYMBOL(sock_rfree);
- void sock_efree(struct sk_buff *skb)
- {
- sock_put(skb->sk);
- }
- EXPORT_SYMBOL(sock_efree);
- kuid_t sock_i_uid(struct sock *sk)
- {
- kuid_t uid;
- read_lock_bh(&sk->sk_callback_lock);
- uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
- read_unlock_bh(&sk->sk_callback_lock);
- return uid;
- }
- EXPORT_SYMBOL(sock_i_uid);
- unsigned long sock_i_ino(struct sock *sk)
- {
- unsigned long ino;
- read_lock_bh(&sk->sk_callback_lock);
- ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
- read_unlock_bh(&sk->sk_callback_lock);
- return ino;
- }
- EXPORT_SYMBOL(sock_i_ino);
- struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
- gfp_t priority)
- {
- if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
- struct sk_buff *skb = alloc_skb(size, priority);
- if (skb) {
- skb_set_owner_w(skb, sk);
- return skb;
- }
- }
- return NULL;
- }
- EXPORT_SYMBOL(sock_wmalloc);
- void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
- {
- if ((unsigned int)size <= sysctl_optmem_max &&
- atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
- void *mem;
-
- atomic_add(size, &sk->sk_omem_alloc);
- mem = kmalloc(size, priority);
- if (mem)
- return mem;
- atomic_sub(size, &sk->sk_omem_alloc);
- }
- return NULL;
- }
- EXPORT_SYMBOL(sock_kmalloc);
- static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
- const bool nullify)
- {
- if (WARN_ON_ONCE(!mem))
- return;
- if (nullify)
- kzfree(mem);
- else
- kfree(mem);
- atomic_sub(size, &sk->sk_omem_alloc);
- }
- void sock_kfree_s(struct sock *sk, void *mem, int size)
- {
- __sock_kfree_s(sk, mem, size, false);
- }
- EXPORT_SYMBOL(sock_kfree_s);
- void sock_kzfree_s(struct sock *sk, void *mem, int size)
- {
- __sock_kfree_s(sk, mem, size, true);
- }
- EXPORT_SYMBOL(sock_kzfree_s);
- static long sock_wait_for_wmem(struct sock *sk, long timeo)
- {
- DEFINE_WAIT(wait);
- sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
- for (;;) {
- if (!timeo)
- break;
- if (signal_pending(current))
- break;
- set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
- prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
- if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
- break;
- if (sk->sk_shutdown & SEND_SHUTDOWN)
- break;
- if (sk->sk_err)
- break;
- timeo = schedule_timeout(timeo);
- }
- finish_wait(sk_sleep(sk), &wait);
- return timeo;
- }
- struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
- unsigned long data_len, int noblock,
- int *errcode, int max_page_order)
- {
- struct sk_buff *skb;
- long timeo;
- int err;
- timeo = sock_sndtimeo(sk, noblock);
- for (;;) {
- err = sock_error(sk);
- if (err != 0)
- goto failure;
- err = -EPIPE;
- if (sk->sk_shutdown & SEND_SHUTDOWN)
- goto failure;
- if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
- break;
- sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
- set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
- err = -EAGAIN;
- if (!timeo)
- goto failure;
- if (signal_pending(current))
- goto interrupted;
- timeo = sock_wait_for_wmem(sk, timeo);
- }
- skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
- errcode, sk->sk_allocation);
- if (skb)
- skb_set_owner_w(skb, sk);
- return skb;
- interrupted:
- err = sock_intr_errno(timeo);
- failure:
- *errcode = err;
- return NULL;
- }
- EXPORT_SYMBOL(sock_alloc_send_pskb);
- struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
- int noblock, int *errcode)
- {
- return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
- }
- EXPORT_SYMBOL(sock_alloc_send_skb);
- int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
- struct sockcm_cookie *sockc)
- {
- u32 tsflags;
- switch (cmsg->cmsg_type) {
- case SO_MARK:
- if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
- return -EPERM;
- if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
- return -EINVAL;
- sockc->mark = *(u32 *)CMSG_DATA(cmsg);
- break;
- case SO_TIMESTAMPING:
- if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
- return -EINVAL;
- tsflags = *(u32 *)CMSG_DATA(cmsg);
- if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
- return -EINVAL;
- sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
- sockc->tsflags |= tsflags;
- break;
-
- case SCM_RIGHTS:
- case SCM_CREDENTIALS:
- break;
- default:
- return -EINVAL;
- }
- return 0;
- }
- EXPORT_SYMBOL(__sock_cmsg_send);
- int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
- struct sockcm_cookie *sockc)
- {
- struct cmsghdr *cmsg;
- int ret;
- for_each_cmsghdr(cmsg, msg) {
- if (!CMSG_OK(msg, cmsg))
- return -EINVAL;
- if (cmsg->cmsg_level != SOL_SOCKET)
- continue;
- ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
- if (ret)
- return ret;
- }
- return 0;
- }
- EXPORT_SYMBOL(sock_cmsg_send);
- #define SKB_FRAG_PAGE_ORDER get_order(32768)
- bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
- {
- if (pfrag->page) {
- if (page_ref_count(pfrag->page) == 1) {
- pfrag->offset = 0;
- return true;
- }
- if (pfrag->offset + sz <= pfrag->size)
- return true;
- put_page(pfrag->page);
- }
- pfrag->offset = 0;
- if (SKB_FRAG_PAGE_ORDER) {
-
- pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
- __GFP_COMP | __GFP_NOWARN |
- __GFP_NORETRY,
- SKB_FRAG_PAGE_ORDER);
- if (likely(pfrag->page)) {
- pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
- return true;
- }
- }
- pfrag->page = alloc_page(gfp);
- if (likely(pfrag->page)) {
- pfrag->size = PAGE_SIZE;
- return true;
- }
- return false;
- }
- EXPORT_SYMBOL(skb_page_frag_refill);
- bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
- {
- if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
- return true;
- sk_enter_memory_pressure(sk);
- sk_stream_moderate_sndbuf(sk);
- return false;
- }
- EXPORT_SYMBOL(sk_page_frag_refill);
- static void __lock_sock(struct sock *sk)
- __releases(&sk->sk_lock.slock)
- __acquires(&sk->sk_lock.slock)
- {
- DEFINE_WAIT(wait);
- for (;;) {
- prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
- TASK_UNINTERRUPTIBLE);
- spin_unlock_bh(&sk->sk_lock.slock);
- schedule();
- spin_lock_bh(&sk->sk_lock.slock);
- if (!sock_owned_by_user(sk))
- break;
- }
- finish_wait(&sk->sk_lock.wq, &wait);
- }
- static void __release_sock(struct sock *sk)
- __releases(&sk->sk_lock.slock)
- __acquires(&sk->sk_lock.slock)
- {
- struct sk_buff *skb, *next;
- while ((skb = sk->sk_backlog.head) != NULL) {
- sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
- spin_unlock_bh(&sk->sk_lock.slock);
- do {
- next = skb->next;
- prefetch(next);
- WARN_ON_ONCE(skb_dst_is_noref(skb));
- skb->next = NULL;
- sk_backlog_rcv(sk, skb);
- cond_resched();
- skb = next;
- } while (skb != NULL);
- spin_lock_bh(&sk->sk_lock.slock);
- }
-
- sk->sk_backlog.len = 0;
- }
- void __sk_flush_backlog(struct sock *sk)
- {
- spin_lock_bh(&sk->sk_lock.slock);
- __release_sock(sk);
- spin_unlock_bh(&sk->sk_lock.slock);
- }
- int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
- {
- int rc;
- DEFINE_WAIT(wait);
- prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
- sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
- rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb);
- sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
- finish_wait(sk_sleep(sk), &wait);
- return rc;
- }
- EXPORT_SYMBOL(sk_wait_data);
- int __sk_mem_schedule(struct sock *sk, int size, int kind)
- {
- struct proto *prot = sk->sk_prot;
- int amt = sk_mem_pages(size);
- long allocated;
- sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
- allocated = sk_memory_allocated_add(sk, amt);
- if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
- !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
- goto suppress_allocation;
-
- if (allocated <= sk_prot_mem_limits(sk, 0)) {
- sk_leave_memory_pressure(sk);
- return 1;
- }
-
- if (allocated > sk_prot_mem_limits(sk, 1))
- sk_enter_memory_pressure(sk);
-
- if (allocated > sk_prot_mem_limits(sk, 2))
- goto suppress_allocation;
-
- if (kind == SK_MEM_RECV) {
- if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
- return 1;
- } else {
- if (sk->sk_type == SOCK_STREAM) {
- if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
- return 1;
- } else if (atomic_read(&sk->sk_wmem_alloc) <
- prot->sysctl_wmem[0])
- return 1;
- }
- if (sk_has_memory_pressure(sk)) {
- int alloc;
- if (!sk_under_memory_pressure(sk))
- return 1;
- alloc = sk_sockets_allocated_read_positive(sk);
- if (sk_prot_mem_limits(sk, 2) > alloc *
- sk_mem_pages(sk->sk_wmem_queued +
- atomic_read(&sk->sk_rmem_alloc) +
- sk->sk_forward_alloc))
- return 1;
- }
- suppress_allocation:
- if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
- sk_stream_moderate_sndbuf(sk);
-
- if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
- return 1;
- }
- trace_sock_exceed_buf_limit(sk, prot, allocated);
-
- sk->sk_forward_alloc -= amt * SK_MEM_QUANTUM;
- sk_memory_allocated_sub(sk, amt);
- if (mem_cgroup_sockets_enabled && sk->sk_memcg)
- mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
- return 0;
- }
- EXPORT_SYMBOL(__sk_mem_schedule);
- void __sk_mem_reclaim(struct sock *sk, int amount)
- {
- amount >>= SK_MEM_QUANTUM_SHIFT;
- sk_memory_allocated_sub(sk, amount);
- sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
- if (mem_cgroup_sockets_enabled && sk->sk_memcg)
- mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
- if (sk_under_memory_pressure(sk) &&
- (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
- sk_leave_memory_pressure(sk);
- }
- EXPORT_SYMBOL(__sk_mem_reclaim);
- int sk_set_peek_off(struct sock *sk, int val)
- {
- if (val < 0)
- return -EINVAL;
- sk->sk_peek_off = val;
- return 0;
- }
- EXPORT_SYMBOL_GPL(sk_set_peek_off);
- int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_bind);
- int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
- int len, int flags)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_connect);
- int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_socketpair);
- int sock_no_accept(struct socket *sock, struct socket *newsock, int flags)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_accept);
- int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
- int *len, int peer)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_getname);
- unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
- {
- return 0;
- }
- EXPORT_SYMBOL(sock_no_poll);
- int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_ioctl);
- int sock_no_listen(struct socket *sock, int backlog)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_listen);
- int sock_no_shutdown(struct socket *sock, int how)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_shutdown);
- int sock_no_setsockopt(struct socket *sock, int level, int optname,
- char __user *optval, unsigned int optlen)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_setsockopt);
- int sock_no_getsockopt(struct socket *sock, int level, int optname,
- char __user *optval, int __user *optlen)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_getsockopt);
- int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_sendmsg);
- int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
- int flags)
- {
- return -EOPNOTSUPP;
- }
- EXPORT_SYMBOL(sock_no_recvmsg);
- int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
- {
-
- return -ENODEV;
- }
- EXPORT_SYMBOL(sock_no_mmap);
- ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
- {
- ssize_t res;
- struct msghdr msg = {.msg_flags = flags};
- struct kvec iov;
- char *kaddr = kmap(page);
- iov.iov_base = kaddr + offset;
- iov.iov_len = size;
- res = kernel_sendmsg(sock, &msg, &iov, 1, size);
- kunmap(page);
- return res;
- }
- EXPORT_SYMBOL(sock_no_sendpage);
- static void sock_def_wakeup(struct sock *sk)
- {
- struct socket_wq *wq;
- rcu_read_lock();
- wq = rcu_dereference(sk->sk_wq);
- if (skwq_has_sleeper(wq))
- wake_up_interruptible_all(&wq->wait);
- rcu_read_unlock();
- }
- static void sock_def_error_report(struct sock *sk)
- {
- struct socket_wq *wq;
- rcu_read_lock();
- wq = rcu_dereference(sk->sk_wq);
- if (skwq_has_sleeper(wq))
- wake_up_interruptible_poll(&wq->wait, POLLERR);
- sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
- rcu_read_unlock();
- }
- static void sock_def_readable(struct sock *sk)
- {
- struct socket_wq *wq;
- rcu_read_lock();
- wq = rcu_dereference(sk->sk_wq);
- if (skwq_has_sleeper(wq))
- wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
- POLLRDNORM | POLLRDBAND);
- sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
- rcu_read_unlock();
- }
- static void sock_def_write_space(struct sock *sk)
- {
- struct socket_wq *wq;
- rcu_read_lock();
-
- if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
- wq = rcu_dereference(sk->sk_wq);
- if (skwq_has_sleeper(wq))
- wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
- POLLWRNORM | POLLWRBAND);
-
- if (sock_writeable(sk))
- sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
- }
- rcu_read_unlock();
- }
- static void sock_def_destruct(struct sock *sk)
- {
- }
- void sk_send_sigurg(struct sock *sk)
- {
- if (sk->sk_socket && sk->sk_socket->file)
- if (send_sigurg(&sk->sk_socket->file->f_owner))
- sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
- }
- EXPORT_SYMBOL(sk_send_sigurg);
- void sk_reset_timer(struct sock *sk, struct timer_list* timer,
- unsigned long expires)
- {
- if (!mod_timer(timer, expires))
- sock_hold(sk);
- }
- EXPORT_SYMBOL(sk_reset_timer);
- void sk_stop_timer(struct sock *sk, struct timer_list* timer)
- {
- if (del_timer(timer))
- __sock_put(sk);
- }
- EXPORT_SYMBOL(sk_stop_timer);
- void sock_init_data(struct socket *sock, struct sock *sk)
- {
- skb_queue_head_init(&sk->sk_receive_queue);
- skb_queue_head_init(&sk->sk_write_queue);
- skb_queue_head_init(&sk->sk_error_queue);
- sk->sk_send_head = NULL;
- init_timer(&sk->sk_timer);
- sk->sk_allocation = GFP_KERNEL;
- sk->sk_rcvbuf = sysctl_rmem_default;
- sk->sk_sndbuf = sysctl_wmem_default;
- sk->sk_state = TCP_CLOSE;
- sk_set_socket(sk, sock);
- sock_set_flag(sk, SOCK_ZAPPED);
- if (sock) {
- sk->sk_type = sock->type;
- sk->sk_wq = sock->wq;
- sock->sk = sk;
- sk->sk_uid = SOCK_INODE(sock)->i_uid;
- } else {
- sk->sk_wq = NULL;
- sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
- }
- rwlock_init(&sk->sk_callback_lock);
- lockdep_set_class_and_name(&sk->sk_callback_lock,
- af_callback_keys + sk->sk_family,
- af_family_clock_key_strings[sk->sk_family]);
- sk->sk_state_change = sock_def_wakeup;
- sk->sk_data_ready = sock_def_readable;
- sk->sk_write_space = sock_def_write_space;
- sk->sk_error_report = sock_def_error_report;
- sk->sk_destruct = sock_def_destruct;
- sk->sk_frag.page = NULL;
- sk->sk_frag.offset = 0;
- sk->sk_peek_off = -1;
- sk->sk_peer_pid = NULL;
- sk->sk_peer_cred = NULL;
- sk->sk_write_pending = 0;
- sk->sk_rcvlowat = 1;
- sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
- sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
- sk->sk_stamp = ktime_set(-1L, 0);
- #if BITS_PER_LONG==32
- seqlock_init(&sk->sk_stamp_seq);
- #endif
- #ifdef CONFIG_NET_RX_BUSY_POLL
- sk->sk_napi_id = 0;
- sk->sk_ll_usec = sysctl_net_busy_read;
- #endif
- sk->sk_max_pacing_rate = ~0U;
- sk->sk_pacing_rate = ~0U;
- sk->sk_incoming_cpu = -1;
-
- smp_wmb();
- atomic_set(&sk->sk_refcnt, 1);
- atomic_set(&sk->sk_drops, 0);
- }
- EXPORT_SYMBOL(sock_init_data);
- void lock_sock_nested(struct sock *sk, int subclass)
- {
- might_sleep();
- spin_lock_bh(&sk->sk_lock.slock);
- if (sk->sk_lock.owned)
- __lock_sock(sk);
- sk->sk_lock.owned = 1;
- spin_unlock(&sk->sk_lock.slock);
-
- mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
- local_bh_enable();
- }
- EXPORT_SYMBOL(lock_sock_nested);
- void release_sock(struct sock *sk)
- {
- spin_lock_bh(&sk->sk_lock.slock);
- if (sk->sk_backlog.tail)
- __release_sock(sk);
-
- if (sk->sk_prot->release_cb)
- sk->sk_prot->release_cb(sk);
- sock_release_ownership(sk);
- if (waitqueue_active(&sk->sk_lock.wq))
- wake_up(&sk->sk_lock.wq);
- spin_unlock_bh(&sk->sk_lock.slock);
- }
- EXPORT_SYMBOL(release_sock);
- bool lock_sock_fast(struct sock *sk)
- {
- might_sleep();
- spin_lock_bh(&sk->sk_lock.slock);
- if (!sk->sk_lock.owned)
-
- return false;
- __lock_sock(sk);
- sk->sk_lock.owned = 1;
- spin_unlock(&sk->sk_lock.slock);
-
- mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
- local_bh_enable();
- return true;
- }
- EXPORT_SYMBOL(lock_sock_fast);
- int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
- {
- struct timeval tv;
- if (!sock_flag(sk, SOCK_TIMESTAMP))
- sock_enable_timestamp(sk, SOCK_TIMESTAMP);
- tv = ktime_to_timeval(sk->sk_stamp);
- if (tv.tv_sec == -1)
- return -ENOENT;
- if (tv.tv_sec == 0) {
- sk->sk_stamp = ktime_get_real();
- tv = ktime_to_timeval(sk->sk_stamp);
- }
- return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
- }
- EXPORT_SYMBOL(sock_get_timestamp);
- int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
- {
- struct timespec ts;
- if (!sock_flag(sk, SOCK_TIMESTAMP))
- sock_enable_timestamp(sk, SOCK_TIMESTAMP);
- ts = ktime_to_timespec(sk->sk_stamp);
- if (ts.tv_sec == -1)
- return -ENOENT;
- if (ts.tv_sec == 0) {
- sk->sk_stamp = ktime_get_real();
- ts = ktime_to_timespec(sk->sk_stamp);
- }
- return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
- }
- EXPORT_SYMBOL(sock_get_timestampns);
- void sock_enable_timestamp(struct sock *sk, int flag)
- {
- if (!sock_flag(sk, flag)) {
- unsigned long previous_flags = sk->sk_flags;
- sock_set_flag(sk, flag);
-
- if (sock_needs_netstamp(sk) &&
- !(previous_flags & SK_FLAGS_TIMESTAMP))
- net_enable_timestamp();
- }
- }
- int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
- int level, int type)
- {
- struct sock_exterr_skb *serr;
- struct sk_buff *skb;
- int copied, err;
- err = -EAGAIN;
- skb = sock_dequeue_err_skb(sk);
- if (skb == NULL)
- goto out;
- copied = skb->len;
- if (copied > len) {
- msg->msg_flags |= MSG_TRUNC;
- copied = len;
- }
- err = skb_copy_datagram_msg(skb, 0, msg, copied);
- if (err)
- goto out_free_skb;
- sock_recv_timestamp(msg, sk, skb);
- serr = SKB_EXT_ERR(skb);
- put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
- msg->msg_flags |= MSG_ERRQUEUE;
- err = copied;
- out_free_skb:
- kfree_skb(skb);
- out:
- return err;
- }
- EXPORT_SYMBOL(sock_recv_errqueue);
- int sock_common_getsockopt(struct socket *sock, int level, int optname,
- char __user *optval, int __user *optlen)
- {
- struct sock *sk = sock->sk;
- return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
- }
- EXPORT_SYMBOL(sock_common_getsockopt);
- #ifdef CONFIG_COMPAT
- int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
- char __user *optval, int __user *optlen)
- {
- struct sock *sk = sock->sk;
- if (sk->sk_prot->compat_getsockopt != NULL)
- return sk->sk_prot->compat_getsockopt(sk, level, optname,
- optval, optlen);
- return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
- }
- EXPORT_SYMBOL(compat_sock_common_getsockopt);
- #endif
- int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
- int flags)
- {
- struct sock *sk = sock->sk;
- int addr_len = 0;
- int err;
- err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
- flags & ~MSG_DONTWAIT, &addr_len);
- if (err >= 0)
- msg->msg_namelen = addr_len;
- return err;
- }
- EXPORT_SYMBOL(sock_common_recvmsg);
- int sock_common_setsockopt(struct socket *sock, int level, int optname,
- char __user *optval, unsigned int optlen)
- {
- struct sock *sk = sock->sk;
- return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
- }
- EXPORT_SYMBOL(sock_common_setsockopt);
- #ifdef CONFIG_COMPAT
- int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
- char __user *optval, unsigned int optlen)
- {
- struct sock *sk = sock->sk;
- if (sk->sk_prot->compat_setsockopt != NULL)
- return sk->sk_prot->compat_setsockopt(sk, level, optname,
- optval, optlen);
- return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
- }
- EXPORT_SYMBOL(compat_sock_common_setsockopt);
- #endif
- void sk_common_release(struct sock *sk)
- {
- if (sk->sk_prot->destroy)
- sk->sk_prot->destroy(sk);
-
- sk->sk_prot->unhash(sk);
-
- sock_orphan(sk);
- xfrm_sk_free_policy(sk);
- sk_refcnt_debug_release(sk);
- sock_put(sk);
- }
- EXPORT_SYMBOL(sk_common_release);
- #ifdef CONFIG_PROC_FS
- #define PROTO_INUSE_NR 64
- struct prot_inuse {
- int val[PROTO_INUSE_NR];
- };
- static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
- #ifdef CONFIG_NET_NS
- void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
- {
- __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
- }
- EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
- int sock_prot_inuse_get(struct net *net, struct proto *prot)
- {
- int cpu, idx = prot->inuse_idx;
- int res = 0;
- for_each_possible_cpu(cpu)
- res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
- return res >= 0 ? res : 0;
- }
- EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
- static int __net_init sock_inuse_init_net(struct net *net)
- {
- net->core.inuse = alloc_percpu(struct prot_inuse);
- return net->core.inuse ? 0 : -ENOMEM;
- }
- static void __net_exit sock_inuse_exit_net(struct net *net)
- {
- free_percpu(net->core.inuse);
- }
- static struct pernet_operations net_inuse_ops = {
- .init = sock_inuse_init_net,
- .exit = sock_inuse_exit_net,
- };
- static __init int net_inuse_init(void)
- {
- if (register_pernet_subsys(&net_inuse_ops))
- panic("Cannot initialize net inuse counters");
- return 0;
- }
- core_initcall(net_inuse_init);
- #else
- static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
- void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
- {
- __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
- }
- EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
- int sock_prot_inuse_get(struct net *net, struct proto *prot)
- {
- int cpu, idx = prot->inuse_idx;
- int res = 0;
- for_each_possible_cpu(cpu)
- res += per_cpu(prot_inuse, cpu).val[idx];
- return res >= 0 ? res : 0;
- }
- EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
- #endif
- static void assign_proto_idx(struct proto *prot)
- {
- prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
- if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
- pr_err("PROTO_INUSE_NR exhausted\n");
- return;
- }
- set_bit(prot->inuse_idx, proto_inuse_idx);
- }
- static void release_proto_idx(struct proto *prot)
- {
- if (prot->inuse_idx != PROTO_INUSE_NR - 1)
- clear_bit(prot->inuse_idx, proto_inuse_idx);
- }
- #else
- static inline void assign_proto_idx(struct proto *prot)
- {
- }
- static inline void release_proto_idx(struct proto *prot)
- {
- }
- #endif
- static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
- {
- if (!rsk_prot)
- return;
- kfree(rsk_prot->slab_name);
- rsk_prot->slab_name = NULL;
- kmem_cache_destroy(rsk_prot->slab);
- rsk_prot->slab = NULL;
- }
- static int req_prot_init(const struct proto *prot)
- {
- struct request_sock_ops *rsk_prot = prot->rsk_prot;
- if (!rsk_prot)
- return 0;
- rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
- prot->name);
- if (!rsk_prot->slab_name)
- return -ENOMEM;
- rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
- rsk_prot->obj_size, 0,
- prot->slab_flags, NULL);
- if (!rsk_prot->slab) {
- pr_crit("%s: Can't create request sock SLAB cache!\n",
- prot->name);
- return -ENOMEM;
- }
- return 0;
- }
- int proto_register(struct proto *prot, int alloc_slab)
- {
- if (alloc_slab) {
- prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
- SLAB_HWCACHE_ALIGN | prot->slab_flags,
- NULL);
- if (prot->slab == NULL) {
- pr_crit("%s: Can't create sock SLAB cache!\n",
- prot->name);
- goto out;
- }
- if (req_prot_init(prot))
- goto out_free_request_sock_slab;
- if (prot->twsk_prot != NULL) {
- prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
- if (prot->twsk_prot->twsk_slab_name == NULL)
- goto out_free_request_sock_slab;
- prot->twsk_prot->twsk_slab =
- kmem_cache_create(prot->twsk_prot->twsk_slab_name,
- prot->twsk_prot->twsk_obj_size,
- 0,
- prot->slab_flags,
- NULL);
- if (prot->twsk_prot->twsk_slab == NULL)
- goto out_free_timewait_sock_slab_name;
- }
- }
- mutex_lock(&proto_list_mutex);
- list_add(&prot->node, &proto_list);
- assign_proto_idx(prot);
- mutex_unlock(&proto_list_mutex);
- return 0;
- out_free_timewait_sock_slab_name:
- kfree(prot->twsk_prot->twsk_slab_name);
- out_free_request_sock_slab:
- req_prot_cleanup(prot->rsk_prot);
- kmem_cache_destroy(prot->slab);
- prot->slab = NULL;
- out:
- return -ENOBUFS;
- }
- EXPORT_SYMBOL(proto_register);
- void proto_unregister(struct proto *prot)
- {
- mutex_lock(&proto_list_mutex);
- release_proto_idx(prot);
- list_del(&prot->node);
- mutex_unlock(&proto_list_mutex);
- kmem_cache_destroy(prot->slab);
- prot->slab = NULL;
- req_prot_cleanup(prot->rsk_prot);
- if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
- kmem_cache_destroy(prot->twsk_prot->twsk_slab);
- kfree(prot->twsk_prot->twsk_slab_name);
- prot->twsk_prot->twsk_slab = NULL;
- }
- }
- EXPORT_SYMBOL(proto_unregister);
- #ifdef CONFIG_PROC_FS
- static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
- __acquires(proto_list_mutex)
- {
- mutex_lock(&proto_list_mutex);
- return seq_list_start_head(&proto_list, *pos);
- }
- static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
- {
- return seq_list_next(v, &proto_list, pos);
- }
- static void proto_seq_stop(struct seq_file *seq, void *v)
- __releases(proto_list_mutex)
- {
- mutex_unlock(&proto_list_mutex);
- }
- static char proto_method_implemented(const void *method)
- {
- return method == NULL ? 'n' : 'y';
- }
- static long sock_prot_memory_allocated(struct proto *proto)
- {
- return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
- }
- static char *sock_prot_memory_pressure(struct proto *proto)
- {
- return proto->memory_pressure != NULL ?
- proto_memory_pressure(proto) ? "yes" : "no" : "NI";
- }
- static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
- {
- seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
- "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
- proto->name,
- proto->obj_size,
- sock_prot_inuse_get(seq_file_net(seq), proto),
- sock_prot_memory_allocated(proto),
- sock_prot_memory_pressure(proto),
- proto->max_header,
- proto->slab == NULL ? "no" : "yes",
- module_name(proto->owner),
- proto_method_implemented(proto->close),
- proto_method_implemented(proto->connect),
- proto_method_implemented(proto->disconnect),
- proto_method_implemented(proto->accept),
- proto_method_implemented(proto->ioctl),
- proto_method_implemented(proto->init),
- proto_method_implemented(proto->destroy),
- proto_method_implemented(proto->shutdown),
- proto_method_implemented(proto->setsockopt),
- proto_method_implemented(proto->getsockopt),
- proto_method_implemented(proto->sendmsg),
- proto_method_implemented(proto->recvmsg),
- proto_method_implemented(proto->sendpage),
- proto_method_implemented(proto->bind),
- proto_method_implemented(proto->backlog_rcv),
- proto_method_implemented(proto->hash),
- proto_method_implemented(proto->unhash),
- proto_method_implemented(proto->get_port),
- proto_method_implemented(proto->enter_memory_pressure));
- }
- static int proto_seq_show(struct seq_file *seq, void *v)
- {
- if (v == &proto_list)
- seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
- "protocol",
- "size",
- "sockets",
- "memory",
- "press",
- "maxhdr",
- "slab",
- "module",
- "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
- else
- proto_seq_printf(seq, list_entry(v, struct proto, node));
- return 0;
- }
- static const struct seq_operations proto_seq_ops = {
- .start = proto_seq_start,
- .next = proto_seq_next,
- .stop = proto_seq_stop,
- .show = proto_seq_show,
- };
- static int proto_seq_open(struct inode *inode, struct file *file)
- {
- return seq_open_net(inode, file, &proto_seq_ops,
- sizeof(struct seq_net_private));
- }
- static const struct file_operations proto_seq_fops = {
- .owner = THIS_MODULE,
- .open = proto_seq_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release_net,
- };
- static __net_init int proto_init_net(struct net *net)
- {
- if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
- return -ENOMEM;
- return 0;
- }
- static __net_exit void proto_exit_net(struct net *net)
- {
- remove_proc_entry("protocols", net->proc_net);
- }
- static __net_initdata struct pernet_operations proto_net_ops = {
- .init = proto_init_net,
- .exit = proto_exit_net,
- };
- static int __init proto_init(void)
- {
- return register_pernet_subsys(&proto_net_ops);
- }
- subsys_initcall(proto_init);
- #endif
|