svc.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518
  1. /*
  2. * linux/net/sunrpc/svc.c
  3. *
  4. * High-level RPC service routines
  5. *
  6. * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
  7. *
  8. * Multiple threads pools and NUMAisation
  9. * Copyright (c) 2006 Silicon Graphics, Inc.
  10. * by Greg Banks <gnb@melbourne.sgi.com>
  11. */
  12. #include <linux/linkage.h>
  13. #include <linux/sched.h>
  14. #include <linux/errno.h>
  15. #include <linux/net.h>
  16. #include <linux/in.h>
  17. #include <linux/mm.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/module.h>
  20. #include <linux/kthread.h>
  21. #include <linux/slab.h>
  22. #include <linux/sunrpc/types.h>
  23. #include <linux/sunrpc/xdr.h>
  24. #include <linux/sunrpc/stats.h>
  25. #include <linux/sunrpc/svcsock.h>
  26. #include <linux/sunrpc/clnt.h>
  27. #include <linux/sunrpc/bc_xprt.h>
  28. #include <trace/events/sunrpc.h>
  29. #define RPCDBG_FACILITY RPCDBG_SVCDSP
  30. static void svc_unregister(const struct svc_serv *serv, struct net *net);
  31. #define svc_serv_is_pooled(serv) ((serv)->sv_ops->svo_function)
  32. #define SVC_POOL_DEFAULT SVC_POOL_GLOBAL
  33. /*
  34. * Structure for mapping cpus to pools and vice versa.
  35. * Setup once during sunrpc initialisation.
  36. */
  37. struct svc_pool_map svc_pool_map = {
  38. .mode = SVC_POOL_DEFAULT
  39. };
  40. EXPORT_SYMBOL_GPL(svc_pool_map);
  41. static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
  42. static int
  43. param_set_pool_mode(const char *val, const struct kernel_param *kp)
  44. {
  45. int *ip = (int *)kp->arg;
  46. struct svc_pool_map *m = &svc_pool_map;
  47. int err;
  48. mutex_lock(&svc_pool_map_mutex);
  49. err = -EBUSY;
  50. if (m->count)
  51. goto out;
  52. err = 0;
  53. if (!strncmp(val, "auto", 4))
  54. *ip = SVC_POOL_AUTO;
  55. else if (!strncmp(val, "global", 6))
  56. *ip = SVC_POOL_GLOBAL;
  57. else if (!strncmp(val, "percpu", 6))
  58. *ip = SVC_POOL_PERCPU;
  59. else if (!strncmp(val, "pernode", 7))
  60. *ip = SVC_POOL_PERNODE;
  61. else
  62. err = -EINVAL;
  63. out:
  64. mutex_unlock(&svc_pool_map_mutex);
  65. return err;
  66. }
  67. static int
  68. param_get_pool_mode(char *buf, const struct kernel_param *kp)
  69. {
  70. int *ip = (int *)kp->arg;
  71. switch (*ip)
  72. {
  73. case SVC_POOL_AUTO:
  74. return strlcpy(buf, "auto", 20);
  75. case SVC_POOL_GLOBAL:
  76. return strlcpy(buf, "global", 20);
  77. case SVC_POOL_PERCPU:
  78. return strlcpy(buf, "percpu", 20);
  79. case SVC_POOL_PERNODE:
  80. return strlcpy(buf, "pernode", 20);
  81. default:
  82. return sprintf(buf, "%d", *ip);
  83. }
  84. }
  85. module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
  86. &svc_pool_map.mode, 0644);
  87. /*
  88. * Detect best pool mapping mode heuristically,
  89. * according to the machine's topology.
  90. */
  91. static int
  92. svc_pool_map_choose_mode(void)
  93. {
  94. unsigned int node;
  95. if (nr_online_nodes > 1) {
  96. /*
  97. * Actually have multiple NUMA nodes,
  98. * so split pools on NUMA node boundaries
  99. */
  100. return SVC_POOL_PERNODE;
  101. }
  102. node = first_online_node;
  103. if (nr_cpus_node(node) > 2) {
  104. /*
  105. * Non-trivial SMP, or CONFIG_NUMA on
  106. * non-NUMA hardware, e.g. with a generic
  107. * x86_64 kernel on Xeons. In this case we
  108. * want to divide the pools on cpu boundaries.
  109. */
  110. return SVC_POOL_PERCPU;
  111. }
  112. /* default: one global pool */
  113. return SVC_POOL_GLOBAL;
  114. }
  115. /*
  116. * Allocate the to_pool[] and pool_to[] arrays.
  117. * Returns 0 on success or an errno.
  118. */
  119. static int
  120. svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
  121. {
  122. m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  123. if (!m->to_pool)
  124. goto fail;
  125. m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  126. if (!m->pool_to)
  127. goto fail_free;
  128. return 0;
  129. fail_free:
  130. kfree(m->to_pool);
  131. m->to_pool = NULL;
  132. fail:
  133. return -ENOMEM;
  134. }
  135. /*
  136. * Initialise the pool map for SVC_POOL_PERCPU mode.
  137. * Returns number of pools or <0 on error.
  138. */
  139. static int
  140. svc_pool_map_init_percpu(struct svc_pool_map *m)
  141. {
  142. unsigned int maxpools = nr_cpu_ids;
  143. unsigned int pidx = 0;
  144. unsigned int cpu;
  145. int err;
  146. err = svc_pool_map_alloc_arrays(m, maxpools);
  147. if (err)
  148. return err;
  149. for_each_online_cpu(cpu) {
  150. BUG_ON(pidx >= maxpools);
  151. m->to_pool[cpu] = pidx;
  152. m->pool_to[pidx] = cpu;
  153. pidx++;
  154. }
  155. /* cpus brought online later all get mapped to pool0, sorry */
  156. return pidx;
  157. };
  158. /*
  159. * Initialise the pool map for SVC_POOL_PERNODE mode.
  160. * Returns number of pools or <0 on error.
  161. */
  162. static int
  163. svc_pool_map_init_pernode(struct svc_pool_map *m)
  164. {
  165. unsigned int maxpools = nr_node_ids;
  166. unsigned int pidx = 0;
  167. unsigned int node;
  168. int err;
  169. err = svc_pool_map_alloc_arrays(m, maxpools);
  170. if (err)
  171. return err;
  172. for_each_node_with_cpus(node) {
  173. /* some architectures (e.g. SN2) have cpuless nodes */
  174. BUG_ON(pidx > maxpools);
  175. m->to_pool[node] = pidx;
  176. m->pool_to[pidx] = node;
  177. pidx++;
  178. }
  179. /* nodes brought online later all get mapped to pool0, sorry */
  180. return pidx;
  181. }
  182. /*
  183. * Add a reference to the global map of cpus to pools (and
  184. * vice versa). Initialise the map if we're the first user.
  185. * Returns the number of pools.
  186. */
  187. unsigned int
  188. svc_pool_map_get(void)
  189. {
  190. struct svc_pool_map *m = &svc_pool_map;
  191. int npools = -1;
  192. mutex_lock(&svc_pool_map_mutex);
  193. if (m->count++) {
  194. mutex_unlock(&svc_pool_map_mutex);
  195. return m->npools;
  196. }
  197. if (m->mode == SVC_POOL_AUTO)
  198. m->mode = svc_pool_map_choose_mode();
  199. switch (m->mode) {
  200. case SVC_POOL_PERCPU:
  201. npools = svc_pool_map_init_percpu(m);
  202. break;
  203. case SVC_POOL_PERNODE:
  204. npools = svc_pool_map_init_pernode(m);
  205. break;
  206. }
  207. if (npools < 0) {
  208. /* default, or memory allocation failure */
  209. npools = 1;
  210. m->mode = SVC_POOL_GLOBAL;
  211. }
  212. m->npools = npools;
  213. mutex_unlock(&svc_pool_map_mutex);
  214. return m->npools;
  215. }
  216. EXPORT_SYMBOL_GPL(svc_pool_map_get);
  217. /*
  218. * Drop a reference to the global map of cpus to pools.
  219. * When the last reference is dropped, the map data is
  220. * freed; this allows the sysadmin to change the pool
  221. * mode using the pool_mode module option without
  222. * rebooting or re-loading sunrpc.ko.
  223. */
  224. void
  225. svc_pool_map_put(void)
  226. {
  227. struct svc_pool_map *m = &svc_pool_map;
  228. mutex_lock(&svc_pool_map_mutex);
  229. if (!--m->count) {
  230. kfree(m->to_pool);
  231. m->to_pool = NULL;
  232. kfree(m->pool_to);
  233. m->pool_to = NULL;
  234. m->npools = 0;
  235. }
  236. mutex_unlock(&svc_pool_map_mutex);
  237. }
  238. EXPORT_SYMBOL_GPL(svc_pool_map_put);
  239. static int svc_pool_map_get_node(unsigned int pidx)
  240. {
  241. const struct svc_pool_map *m = &svc_pool_map;
  242. if (m->count) {
  243. if (m->mode == SVC_POOL_PERCPU)
  244. return cpu_to_node(m->pool_to[pidx]);
  245. if (m->mode == SVC_POOL_PERNODE)
  246. return m->pool_to[pidx];
  247. }
  248. return NUMA_NO_NODE;
  249. }
  250. /*
  251. * Set the given thread's cpus_allowed mask so that it
  252. * will only run on cpus in the given pool.
  253. */
  254. static inline void
  255. svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
  256. {
  257. struct svc_pool_map *m = &svc_pool_map;
  258. unsigned int node = m->pool_to[pidx];
  259. /*
  260. * The caller checks for sv_nrpools > 1, which
  261. * implies that we've been initialized.
  262. */
  263. WARN_ON_ONCE(m->count == 0);
  264. if (m->count == 0)
  265. return;
  266. switch (m->mode) {
  267. case SVC_POOL_PERCPU:
  268. {
  269. set_cpus_allowed_ptr(task, cpumask_of(node));
  270. break;
  271. }
  272. case SVC_POOL_PERNODE:
  273. {
  274. set_cpus_allowed_ptr(task, cpumask_of_node(node));
  275. break;
  276. }
  277. }
  278. }
  279. /*
  280. * Use the mapping mode to choose a pool for a given CPU.
  281. * Used when enqueueing an incoming RPC. Always returns
  282. * a non-NULL pool pointer.
  283. */
  284. struct svc_pool *
  285. svc_pool_for_cpu(struct svc_serv *serv, int cpu)
  286. {
  287. struct svc_pool_map *m = &svc_pool_map;
  288. unsigned int pidx = 0;
  289. /*
  290. * An uninitialised map happens in a pure client when
  291. * lockd is brought up, so silently treat it the
  292. * same as SVC_POOL_GLOBAL.
  293. */
  294. if (svc_serv_is_pooled(serv)) {
  295. switch (m->mode) {
  296. case SVC_POOL_PERCPU:
  297. pidx = m->to_pool[cpu];
  298. break;
  299. case SVC_POOL_PERNODE:
  300. pidx = m->to_pool[cpu_to_node(cpu)];
  301. break;
  302. }
  303. }
  304. return &serv->sv_pools[pidx % serv->sv_nrpools];
  305. }
  306. int svc_rpcb_setup(struct svc_serv *serv, struct net *net)
  307. {
  308. int err;
  309. err = rpcb_create_local(net);
  310. if (err)
  311. return err;
  312. /* Remove any stale portmap registrations */
  313. svc_unregister(serv, net);
  314. return 0;
  315. }
  316. EXPORT_SYMBOL_GPL(svc_rpcb_setup);
  317. void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net)
  318. {
  319. svc_unregister(serv, net);
  320. rpcb_put_local(net);
  321. }
  322. EXPORT_SYMBOL_GPL(svc_rpcb_cleanup);
  323. static int svc_uses_rpcbind(struct svc_serv *serv)
  324. {
  325. struct svc_program *progp;
  326. unsigned int i;
  327. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  328. for (i = 0; i < progp->pg_nvers; i++) {
  329. if (progp->pg_vers[i] == NULL)
  330. continue;
  331. if (progp->pg_vers[i]->vs_hidden == 0)
  332. return 1;
  333. }
  334. }
  335. return 0;
  336. }
  337. int svc_bind(struct svc_serv *serv, struct net *net)
  338. {
  339. if (!svc_uses_rpcbind(serv))
  340. return 0;
  341. return svc_rpcb_setup(serv, net);
  342. }
  343. EXPORT_SYMBOL_GPL(svc_bind);
  344. #if defined(CONFIG_SUNRPC_BACKCHANNEL)
  345. static void
  346. __svc_init_bc(struct svc_serv *serv)
  347. {
  348. INIT_LIST_HEAD(&serv->sv_cb_list);
  349. spin_lock_init(&serv->sv_cb_lock);
  350. init_waitqueue_head(&serv->sv_cb_waitq);
  351. }
  352. #else
  353. static void
  354. __svc_init_bc(struct svc_serv *serv)
  355. {
  356. }
  357. #endif
  358. /*
  359. * Create an RPC service
  360. */
  361. static struct svc_serv *
  362. __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
  363. struct svc_serv_ops *ops)
  364. {
  365. struct svc_serv *serv;
  366. unsigned int vers;
  367. unsigned int xdrsize;
  368. unsigned int i;
  369. if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
  370. return NULL;
  371. serv->sv_name = prog->pg_name;
  372. serv->sv_program = prog;
  373. serv->sv_nrthreads = 1;
  374. serv->sv_stats = prog->pg_stats;
  375. if (bufsize > RPCSVC_MAXPAYLOAD)
  376. bufsize = RPCSVC_MAXPAYLOAD;
  377. serv->sv_max_payload = bufsize? bufsize : 4096;
  378. serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
  379. serv->sv_ops = ops;
  380. xdrsize = 0;
  381. while (prog) {
  382. prog->pg_lovers = prog->pg_nvers-1;
  383. for (vers=0; vers<prog->pg_nvers ; vers++)
  384. if (prog->pg_vers[vers]) {
  385. prog->pg_hivers = vers;
  386. if (prog->pg_lovers > vers)
  387. prog->pg_lovers = vers;
  388. if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
  389. xdrsize = prog->pg_vers[vers]->vs_xdrsize;
  390. }
  391. prog = prog->pg_next;
  392. }
  393. serv->sv_xdrsize = xdrsize;
  394. INIT_LIST_HEAD(&serv->sv_tempsocks);
  395. INIT_LIST_HEAD(&serv->sv_permsocks);
  396. init_timer(&serv->sv_temptimer);
  397. spin_lock_init(&serv->sv_lock);
  398. __svc_init_bc(serv);
  399. serv->sv_nrpools = npools;
  400. serv->sv_pools =
  401. kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
  402. GFP_KERNEL);
  403. if (!serv->sv_pools) {
  404. kfree(serv);
  405. return NULL;
  406. }
  407. for (i = 0; i < serv->sv_nrpools; i++) {
  408. struct svc_pool *pool = &serv->sv_pools[i];
  409. dprintk("svc: initialising pool %u for %s\n",
  410. i, serv->sv_name);
  411. pool->sp_id = i;
  412. INIT_LIST_HEAD(&pool->sp_sockets);
  413. INIT_LIST_HEAD(&pool->sp_all_threads);
  414. spin_lock_init(&pool->sp_lock);
  415. }
  416. return serv;
  417. }
  418. struct svc_serv *
  419. svc_create(struct svc_program *prog, unsigned int bufsize,
  420. struct svc_serv_ops *ops)
  421. {
  422. return __svc_create(prog, bufsize, /*npools*/1, ops);
  423. }
  424. EXPORT_SYMBOL_GPL(svc_create);
  425. struct svc_serv *
  426. svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
  427. struct svc_serv_ops *ops)
  428. {
  429. struct svc_serv *serv;
  430. unsigned int npools = svc_pool_map_get();
  431. serv = __svc_create(prog, bufsize, npools, ops);
  432. if (!serv)
  433. goto out_err;
  434. return serv;
  435. out_err:
  436. svc_pool_map_put();
  437. return NULL;
  438. }
  439. EXPORT_SYMBOL_GPL(svc_create_pooled);
  440. void svc_shutdown_net(struct svc_serv *serv, struct net *net)
  441. {
  442. svc_close_net(serv, net);
  443. if (serv->sv_ops->svo_shutdown)
  444. serv->sv_ops->svo_shutdown(serv, net);
  445. }
  446. EXPORT_SYMBOL_GPL(svc_shutdown_net);
  447. /*
  448. * Destroy an RPC service. Should be called with appropriate locking to
  449. * protect the sv_nrthreads, sv_permsocks and sv_tempsocks.
  450. */
  451. void
  452. svc_destroy(struct svc_serv *serv)
  453. {
  454. dprintk("svc: svc_destroy(%s, %d)\n",
  455. serv->sv_program->pg_name,
  456. serv->sv_nrthreads);
  457. if (serv->sv_nrthreads) {
  458. if (--(serv->sv_nrthreads) != 0) {
  459. svc_sock_update_bufs(serv);
  460. return;
  461. }
  462. } else
  463. printk("svc_destroy: no threads for serv=%p!\n", serv);
  464. del_timer_sync(&serv->sv_temptimer);
  465. /*
  466. * The last user is gone and thus all sockets have to be destroyed to
  467. * the point. Check this.
  468. */
  469. BUG_ON(!list_empty(&serv->sv_permsocks));
  470. BUG_ON(!list_empty(&serv->sv_tempsocks));
  471. cache_clean_deferred(serv);
  472. if (svc_serv_is_pooled(serv))
  473. svc_pool_map_put();
  474. kfree(serv->sv_pools);
  475. kfree(serv);
  476. }
  477. EXPORT_SYMBOL_GPL(svc_destroy);
  478. /*
  479. * Allocate an RPC server's buffer space.
  480. * We allocate pages and place them in rq_argpages.
  481. */
  482. static int
  483. svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
  484. {
  485. unsigned int pages, arghi;
  486. /* bc_xprt uses fore channel allocated buffers */
  487. if (svc_is_backchannel(rqstp))
  488. return 1;
  489. pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
  490. * We assume one is at most one page
  491. */
  492. arghi = 0;
  493. WARN_ON_ONCE(pages > RPCSVC_MAXPAGES);
  494. if (pages > RPCSVC_MAXPAGES)
  495. pages = RPCSVC_MAXPAGES;
  496. while (pages) {
  497. struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
  498. if (!p)
  499. break;
  500. rqstp->rq_pages[arghi++] = p;
  501. pages--;
  502. }
  503. return pages == 0;
  504. }
  505. /*
  506. * Release an RPC server buffer
  507. */
  508. static void
  509. svc_release_buffer(struct svc_rqst *rqstp)
  510. {
  511. unsigned int i;
  512. for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
  513. if (rqstp->rq_pages[i])
  514. put_page(rqstp->rq_pages[i]);
  515. }
  516. struct svc_rqst *
  517. svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)
  518. {
  519. struct svc_rqst *rqstp;
  520. rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
  521. if (!rqstp)
  522. return rqstp;
  523. __set_bit(RQ_BUSY, &rqstp->rq_flags);
  524. spin_lock_init(&rqstp->rq_lock);
  525. rqstp->rq_server = serv;
  526. rqstp->rq_pool = pool;
  527. rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
  528. if (!rqstp->rq_argp)
  529. goto out_enomem;
  530. rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
  531. if (!rqstp->rq_resp)
  532. goto out_enomem;
  533. if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
  534. goto out_enomem;
  535. return rqstp;
  536. out_enomem:
  537. svc_rqst_free(rqstp);
  538. return NULL;
  539. }
  540. EXPORT_SYMBOL_GPL(svc_rqst_alloc);
  541. struct svc_rqst *
  542. svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
  543. {
  544. struct svc_rqst *rqstp;
  545. rqstp = svc_rqst_alloc(serv, pool, node);
  546. if (!rqstp)
  547. return ERR_PTR(-ENOMEM);
  548. serv->sv_nrthreads++;
  549. spin_lock_bh(&pool->sp_lock);
  550. pool->sp_nrthreads++;
  551. list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
  552. spin_unlock_bh(&pool->sp_lock);
  553. return rqstp;
  554. }
  555. EXPORT_SYMBOL_GPL(svc_prepare_thread);
  556. /*
  557. * Choose a pool in which to create a new thread, for svc_set_num_threads
  558. */
  559. static inline struct svc_pool *
  560. choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  561. {
  562. if (pool != NULL)
  563. return pool;
  564. return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
  565. }
  566. /*
  567. * Choose a thread to kill, for svc_set_num_threads
  568. */
  569. static inline struct task_struct *
  570. choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  571. {
  572. unsigned int i;
  573. struct task_struct *task = NULL;
  574. if (pool != NULL) {
  575. spin_lock_bh(&pool->sp_lock);
  576. } else {
  577. /* choose a pool in round-robin fashion */
  578. for (i = 0; i < serv->sv_nrpools; i++) {
  579. pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
  580. spin_lock_bh(&pool->sp_lock);
  581. if (!list_empty(&pool->sp_all_threads))
  582. goto found_pool;
  583. spin_unlock_bh(&pool->sp_lock);
  584. }
  585. return NULL;
  586. }
  587. found_pool:
  588. if (!list_empty(&pool->sp_all_threads)) {
  589. struct svc_rqst *rqstp;
  590. /*
  591. * Remove from the pool->sp_all_threads list
  592. * so we don't try to kill it again.
  593. */
  594. rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
  595. set_bit(RQ_VICTIM, &rqstp->rq_flags);
  596. list_del_rcu(&rqstp->rq_all);
  597. task = rqstp->rq_task;
  598. }
  599. spin_unlock_bh(&pool->sp_lock);
  600. return task;
  601. }
  602. /* create new threads */
  603. static int
  604. svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  605. {
  606. struct svc_rqst *rqstp;
  607. struct task_struct *task;
  608. struct svc_pool *chosen_pool;
  609. unsigned int state = serv->sv_nrthreads-1;
  610. int node;
  611. do {
  612. nrservs--;
  613. chosen_pool = choose_pool(serv, pool, &state);
  614. node = svc_pool_map_get_node(chosen_pool->sp_id);
  615. rqstp = svc_prepare_thread(serv, chosen_pool, node);
  616. if (IS_ERR(rqstp))
  617. return PTR_ERR(rqstp);
  618. __module_get(serv->sv_ops->svo_module);
  619. task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
  620. node, "%s", serv->sv_name);
  621. if (IS_ERR(task)) {
  622. module_put(serv->sv_ops->svo_module);
  623. svc_exit_thread(rqstp);
  624. return PTR_ERR(task);
  625. }
  626. rqstp->rq_task = task;
  627. if (serv->sv_nrpools > 1)
  628. svc_pool_map_set_cpumask(task, chosen_pool->sp_id);
  629. svc_sock_update_bufs(serv);
  630. wake_up_process(task);
  631. } while (nrservs > 0);
  632. return 0;
  633. }
  634. /* destroy old threads */
  635. static int
  636. svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  637. {
  638. struct task_struct *task;
  639. unsigned int state = serv->sv_nrthreads-1;
  640. /* destroy old threads */
  641. do {
  642. task = choose_victim(serv, pool, &state);
  643. if (task == NULL)
  644. break;
  645. send_sig(SIGINT, task, 1);
  646. nrservs++;
  647. } while (nrservs < 0);
  648. return 0;
  649. }
  650. /*
  651. * Create or destroy enough new threads to make the number
  652. * of threads the given number. If `pool' is non-NULL, applies
  653. * only to threads in that pool, otherwise round-robins between
  654. * all pools. Caller must ensure that mutual exclusion between this and
  655. * server startup or shutdown.
  656. *
  657. * Destroying threads relies on the service threads filling in
  658. * rqstp->rq_task, which only the nfs ones do. Assumes the serv
  659. * has been created using svc_create_pooled().
  660. *
  661. * Based on code that used to be in nfsd_svc() but tweaked
  662. * to be pool-aware.
  663. */
  664. int
  665. svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  666. {
  667. if (pool == NULL) {
  668. /* The -1 assumes caller has done a svc_get() */
  669. nrservs -= (serv->sv_nrthreads-1);
  670. } else {
  671. spin_lock_bh(&pool->sp_lock);
  672. nrservs -= pool->sp_nrthreads;
  673. spin_unlock_bh(&pool->sp_lock);
  674. }
  675. if (nrservs > 0)
  676. return svc_start_kthreads(serv, pool, nrservs);
  677. if (nrservs < 0)
  678. return svc_signal_kthreads(serv, pool, nrservs);
  679. return 0;
  680. }
  681. EXPORT_SYMBOL_GPL(svc_set_num_threads);
  682. /* destroy old threads */
  683. static int
  684. svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  685. {
  686. struct task_struct *task;
  687. unsigned int state = serv->sv_nrthreads-1;
  688. /* destroy old threads */
  689. do {
  690. task = choose_victim(serv, pool, &state);
  691. if (task == NULL)
  692. break;
  693. kthread_stop(task);
  694. nrservs++;
  695. } while (nrservs < 0);
  696. return 0;
  697. }
  698. int
  699. svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  700. {
  701. if (pool == NULL) {
  702. /* The -1 assumes caller has done a svc_get() */
  703. nrservs -= (serv->sv_nrthreads-1);
  704. } else {
  705. spin_lock_bh(&pool->sp_lock);
  706. nrservs -= pool->sp_nrthreads;
  707. spin_unlock_bh(&pool->sp_lock);
  708. }
  709. if (nrservs > 0)
  710. return svc_start_kthreads(serv, pool, nrservs);
  711. if (nrservs < 0)
  712. return svc_stop_kthreads(serv, pool, nrservs);
  713. return 0;
  714. }
  715. EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);
  716. /*
  717. * Called from a server thread as it's exiting. Caller must hold the "service
  718. * mutex" for the service.
  719. */
  720. void
  721. svc_rqst_free(struct svc_rqst *rqstp)
  722. {
  723. svc_release_buffer(rqstp);
  724. kfree(rqstp->rq_resp);
  725. kfree(rqstp->rq_argp);
  726. kfree(rqstp->rq_auth_data);
  727. kfree_rcu(rqstp, rq_rcu_head);
  728. }
  729. EXPORT_SYMBOL_GPL(svc_rqst_free);
  730. void
  731. svc_exit_thread(struct svc_rqst *rqstp)
  732. {
  733. struct svc_serv *serv = rqstp->rq_server;
  734. struct svc_pool *pool = rqstp->rq_pool;
  735. spin_lock_bh(&pool->sp_lock);
  736. pool->sp_nrthreads--;
  737. if (!test_and_set_bit(RQ_VICTIM, &rqstp->rq_flags))
  738. list_del_rcu(&rqstp->rq_all);
  739. spin_unlock_bh(&pool->sp_lock);
  740. svc_rqst_free(rqstp);
  741. /* Release the server */
  742. if (serv)
  743. svc_destroy(serv);
  744. }
  745. EXPORT_SYMBOL_GPL(svc_exit_thread);
  746. /*
  747. * Register an "inet" protocol family netid with the local
  748. * rpcbind daemon via an rpcbind v4 SET request.
  749. *
  750. * No netconfig infrastructure is available in the kernel, so
  751. * we map IP_ protocol numbers to netids by hand.
  752. *
  753. * Returns zero on success; a negative errno value is returned
  754. * if any error occurs.
  755. */
  756. static int __svc_rpcb_register4(struct net *net, const u32 program,
  757. const u32 version,
  758. const unsigned short protocol,
  759. const unsigned short port)
  760. {
  761. const struct sockaddr_in sin = {
  762. .sin_family = AF_INET,
  763. .sin_addr.s_addr = htonl(INADDR_ANY),
  764. .sin_port = htons(port),
  765. };
  766. const char *netid;
  767. int error;
  768. switch (protocol) {
  769. case IPPROTO_UDP:
  770. netid = RPCBIND_NETID_UDP;
  771. break;
  772. case IPPROTO_TCP:
  773. netid = RPCBIND_NETID_TCP;
  774. break;
  775. default:
  776. return -ENOPROTOOPT;
  777. }
  778. error = rpcb_v4_register(net, program, version,
  779. (const struct sockaddr *)&sin, netid);
  780. /*
  781. * User space didn't support rpcbind v4, so retry this
  782. * registration request with the legacy rpcbind v2 protocol.
  783. */
  784. if (error == -EPROTONOSUPPORT)
  785. error = rpcb_register(net, program, version, protocol, port);
  786. return error;
  787. }
  788. #if IS_ENABLED(CONFIG_IPV6)
  789. /*
  790. * Register an "inet6" protocol family netid with the local
  791. * rpcbind daemon via an rpcbind v4 SET request.
  792. *
  793. * No netconfig infrastructure is available in the kernel, so
  794. * we map IP_ protocol numbers to netids by hand.
  795. *
  796. * Returns zero on success; a negative errno value is returned
  797. * if any error occurs.
  798. */
  799. static int __svc_rpcb_register6(struct net *net, const u32 program,
  800. const u32 version,
  801. const unsigned short protocol,
  802. const unsigned short port)
  803. {
  804. const struct sockaddr_in6 sin6 = {
  805. .sin6_family = AF_INET6,
  806. .sin6_addr = IN6ADDR_ANY_INIT,
  807. .sin6_port = htons(port),
  808. };
  809. const char *netid;
  810. int error;
  811. switch (protocol) {
  812. case IPPROTO_UDP:
  813. netid = RPCBIND_NETID_UDP6;
  814. break;
  815. case IPPROTO_TCP:
  816. netid = RPCBIND_NETID_TCP6;
  817. break;
  818. default:
  819. return -ENOPROTOOPT;
  820. }
  821. error = rpcb_v4_register(net, program, version,
  822. (const struct sockaddr *)&sin6, netid);
  823. /*
  824. * User space didn't support rpcbind version 4, so we won't
  825. * use a PF_INET6 listener.
  826. */
  827. if (error == -EPROTONOSUPPORT)
  828. error = -EAFNOSUPPORT;
  829. return error;
  830. }
  831. #endif /* IS_ENABLED(CONFIG_IPV6) */
  832. /*
  833. * Register a kernel RPC service via rpcbind version 4.
  834. *
  835. * Returns zero on success; a negative errno value is returned
  836. * if any error occurs.
  837. */
  838. static int __svc_register(struct net *net, const char *progname,
  839. const u32 program, const u32 version,
  840. const int family,
  841. const unsigned short protocol,
  842. const unsigned short port)
  843. {
  844. int error = -EAFNOSUPPORT;
  845. switch (family) {
  846. case PF_INET:
  847. error = __svc_rpcb_register4(net, program, version,
  848. protocol, port);
  849. break;
  850. #if IS_ENABLED(CONFIG_IPV6)
  851. case PF_INET6:
  852. error = __svc_rpcb_register6(net, program, version,
  853. protocol, port);
  854. #endif
  855. }
  856. return error;
  857. }
  858. /**
  859. * svc_register - register an RPC service with the local portmapper
  860. * @serv: svc_serv struct for the service to register
  861. * @net: net namespace for the service to register
  862. * @family: protocol family of service's listener socket
  863. * @proto: transport protocol number to advertise
  864. * @port: port to advertise
  865. *
  866. * Service is registered for any address in the passed-in protocol family
  867. */
  868. int svc_register(const struct svc_serv *serv, struct net *net,
  869. const int family, const unsigned short proto,
  870. const unsigned short port)
  871. {
  872. struct svc_program *progp;
  873. struct svc_version *vers;
  874. unsigned int i;
  875. int error = 0;
  876. WARN_ON_ONCE(proto == 0 && port == 0);
  877. if (proto == 0 && port == 0)
  878. return -EINVAL;
  879. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  880. for (i = 0; i < progp->pg_nvers; i++) {
  881. vers = progp->pg_vers[i];
  882. if (vers == NULL)
  883. continue;
  884. dprintk("svc: svc_register(%sv%d, %s, %u, %u)%s\n",
  885. progp->pg_name,
  886. i,
  887. proto == IPPROTO_UDP? "udp" : "tcp",
  888. port,
  889. family,
  890. vers->vs_hidden ?
  891. " (but not telling portmap)" : "");
  892. if (vers->vs_hidden)
  893. continue;
  894. error = __svc_register(net, progp->pg_name, progp->pg_prog,
  895. i, family, proto, port);
  896. if (vers->vs_rpcb_optnl) {
  897. error = 0;
  898. continue;
  899. }
  900. if (error < 0) {
  901. printk(KERN_WARNING "svc: failed to register "
  902. "%sv%u RPC service (errno %d).\n",
  903. progp->pg_name, i, -error);
  904. break;
  905. }
  906. }
  907. }
  908. return error;
  909. }
  910. /*
  911. * If user space is running rpcbind, it should take the v4 UNSET
  912. * and clear everything for this [program, version]. If user space
  913. * is running portmap, it will reject the v4 UNSET, but won't have
  914. * any "inet6" entries anyway. So a PMAP_UNSET should be sufficient
  915. * in this case to clear all existing entries for [program, version].
  916. */
  917. static void __svc_unregister(struct net *net, const u32 program, const u32 version,
  918. const char *progname)
  919. {
  920. int error;
  921. error = rpcb_v4_register(net, program, version, NULL, "");
  922. /*
  923. * User space didn't support rpcbind v4, so retry this
  924. * request with the legacy rpcbind v2 protocol.
  925. */
  926. if (error == -EPROTONOSUPPORT)
  927. error = rpcb_register(net, program, version, 0, 0);
  928. dprintk("svc: %s(%sv%u), error %d\n",
  929. __func__, progname, version, error);
  930. }
  931. /*
  932. * All netids, bind addresses and ports registered for [program, version]
  933. * are removed from the local rpcbind database (if the service is not
  934. * hidden) to make way for a new instance of the service.
  935. *
  936. * The result of unregistration is reported via dprintk for those who want
  937. * verification of the result, but is otherwise not important.
  938. */
  939. static void svc_unregister(const struct svc_serv *serv, struct net *net)
  940. {
  941. struct svc_program *progp;
  942. unsigned long flags;
  943. unsigned int i;
  944. clear_thread_flag(TIF_SIGPENDING);
  945. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  946. for (i = 0; i < progp->pg_nvers; i++) {
  947. if (progp->pg_vers[i] == NULL)
  948. continue;
  949. if (progp->pg_vers[i]->vs_hidden)
  950. continue;
  951. dprintk("svc: attempting to unregister %sv%u\n",
  952. progp->pg_name, i);
  953. __svc_unregister(net, progp->pg_prog, i, progp->pg_name);
  954. }
  955. }
  956. spin_lock_irqsave(&current->sighand->siglock, flags);
  957. recalc_sigpending();
  958. spin_unlock_irqrestore(&current->sighand->siglock, flags);
  959. }
  960. /*
  961. * dprintk the given error with the address of the client that caused it.
  962. */
  963. #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
  964. static __printf(2, 3)
  965. void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
  966. {
  967. struct va_format vaf;
  968. va_list args;
  969. char buf[RPC_MAX_ADDRBUFLEN];
  970. va_start(args, fmt);
  971. vaf.fmt = fmt;
  972. vaf.va = &args;
  973. dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
  974. va_end(args);
  975. }
  976. #else
  977. static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
  978. #endif
  979. extern void svc_tcp_prep_reply_hdr(struct svc_rqst *);
  980. /*
  981. * Common routine for processing the RPC request.
  982. */
  983. static int
  984. svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
  985. {
  986. struct svc_program *progp;
  987. struct svc_version *versp = NULL; /* compiler food */
  988. struct svc_procedure *procp = NULL;
  989. struct svc_serv *serv = rqstp->rq_server;
  990. kxdrproc_t xdr;
  991. __be32 *statp;
  992. u32 prog, vers, proc;
  993. __be32 auth_stat, rpc_stat;
  994. int auth_res;
  995. __be32 *reply_statp;
  996. rpc_stat = rpc_success;
  997. if (argv->iov_len < 6*4)
  998. goto err_short_len;
  999. /* Will be turned off only in gss privacy case: */
  1000. set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  1001. /* Will be turned off only when NFSv4 Sessions are used */
  1002. set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  1003. clear_bit(RQ_DROPME, &rqstp->rq_flags);
  1004. /* Setup reply header */
  1005. if (rqstp->rq_prot == IPPROTO_TCP)
  1006. svc_tcp_prep_reply_hdr(rqstp);
  1007. svc_putu32(resv, rqstp->rq_xid);
  1008. vers = svc_getnl(argv);
  1009. /* First words of reply: */
  1010. svc_putnl(resv, 1); /* REPLY */
  1011. if (vers != 2) /* RPC version number */
  1012. goto err_bad_rpc;
  1013. /* Save position in case we later decide to reject: */
  1014. reply_statp = resv->iov_base + resv->iov_len;
  1015. svc_putnl(resv, 0); /* ACCEPT */
  1016. rqstp->rq_prog = prog = svc_getnl(argv); /* program number */
  1017. rqstp->rq_vers = vers = svc_getnl(argv); /* version number */
  1018. rqstp->rq_proc = proc = svc_getnl(argv); /* procedure number */
  1019. for (progp = serv->sv_program; progp; progp = progp->pg_next)
  1020. if (prog == progp->pg_prog)
  1021. break;
  1022. /*
  1023. * Decode auth data, and add verifier to reply buffer.
  1024. * We do this before anything else in order to get a decent
  1025. * auth verifier.
  1026. */
  1027. auth_res = svc_authenticate(rqstp, &auth_stat);
  1028. /* Also give the program a chance to reject this call: */
  1029. if (auth_res == SVC_OK && progp) {
  1030. auth_stat = rpc_autherr_badcred;
  1031. auth_res = progp->pg_authenticate(rqstp);
  1032. }
  1033. switch (auth_res) {
  1034. case SVC_OK:
  1035. break;
  1036. case SVC_GARBAGE:
  1037. goto err_garbage;
  1038. case SVC_SYSERR:
  1039. rpc_stat = rpc_system_err;
  1040. goto err_bad;
  1041. case SVC_DENIED:
  1042. goto err_bad_auth;
  1043. case SVC_CLOSE:
  1044. goto close;
  1045. case SVC_DROP:
  1046. goto dropit;
  1047. case SVC_COMPLETE:
  1048. goto sendit;
  1049. }
  1050. if (progp == NULL)
  1051. goto err_bad_prog;
  1052. if (vers >= progp->pg_nvers ||
  1053. !(versp = progp->pg_vers[vers]))
  1054. goto err_bad_vers;
  1055. procp = versp->vs_proc + proc;
  1056. if (proc >= versp->vs_nproc || !procp->pc_func)
  1057. goto err_bad_proc;
  1058. rqstp->rq_procinfo = procp;
  1059. /* Syntactic check complete */
  1060. serv->sv_stats->rpccnt++;
  1061. /* Build the reply header. */
  1062. statp = resv->iov_base +resv->iov_len;
  1063. svc_putnl(resv, RPC_SUCCESS);
  1064. /* Bump per-procedure stats counter */
  1065. procp->pc_count++;
  1066. /* Initialize storage for argp and resp */
  1067. memset(rqstp->rq_argp, 0, procp->pc_argsize);
  1068. memset(rqstp->rq_resp, 0, procp->pc_ressize);
  1069. /* un-reserve some of the out-queue now that we have a
  1070. * better idea of reply size
  1071. */
  1072. if (procp->pc_xdrressize)
  1073. svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
  1074. /* Call the function that processes the request. */
  1075. if (!versp->vs_dispatch) {
  1076. /* Decode arguments */
  1077. xdr = procp->pc_decode;
  1078. if (xdr && !xdr(rqstp, argv->iov_base, rqstp->rq_argp))
  1079. goto err_garbage;
  1080. *statp = procp->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);
  1081. /* Encode reply */
  1082. if (*statp == rpc_drop_reply ||
  1083. test_bit(RQ_DROPME, &rqstp->rq_flags)) {
  1084. if (procp->pc_release)
  1085. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  1086. goto dropit;
  1087. }
  1088. if (*statp == rpc_autherr_badcred) {
  1089. if (procp->pc_release)
  1090. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  1091. goto err_bad_auth;
  1092. }
  1093. if (*statp == rpc_success &&
  1094. (xdr = procp->pc_encode) &&
  1095. !xdr(rqstp, resv->iov_base+resv->iov_len, rqstp->rq_resp)) {
  1096. dprintk("svc: failed to encode reply\n");
  1097. /* serv->sv_stats->rpcsystemerr++; */
  1098. *statp = rpc_system_err;
  1099. }
  1100. } else {
  1101. dprintk("svc: calling dispatcher\n");
  1102. if (!versp->vs_dispatch(rqstp, statp)) {
  1103. /* Release reply info */
  1104. if (procp->pc_release)
  1105. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  1106. goto dropit;
  1107. }
  1108. }
  1109. /* Check RPC status result */
  1110. if (*statp != rpc_success)
  1111. resv->iov_len = ((void*)statp) - resv->iov_base + 4;
  1112. /* Release reply info */
  1113. if (procp->pc_release)
  1114. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  1115. if (procp->pc_encode == NULL)
  1116. goto dropit;
  1117. sendit:
  1118. if (svc_authorise(rqstp))
  1119. goto close;
  1120. return 1; /* Caller can now send it */
  1121. dropit:
  1122. svc_authorise(rqstp); /* doesn't hurt to call this twice */
  1123. dprintk("svc: svc_process dropit\n");
  1124. return 0;
  1125. close:
  1126. if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
  1127. svc_close_xprt(rqstp->rq_xprt);
  1128. dprintk("svc: svc_process close\n");
  1129. return 0;
  1130. err_short_len:
  1131. svc_printk(rqstp, "short len %Zd, dropping request\n",
  1132. argv->iov_len);
  1133. goto close;
  1134. err_bad_rpc:
  1135. serv->sv_stats->rpcbadfmt++;
  1136. svc_putnl(resv, 1); /* REJECT */
  1137. svc_putnl(resv, 0); /* RPC_MISMATCH */
  1138. svc_putnl(resv, 2); /* Only RPCv2 supported */
  1139. svc_putnl(resv, 2);
  1140. goto sendit;
  1141. err_bad_auth:
  1142. dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
  1143. serv->sv_stats->rpcbadauth++;
  1144. /* Restore write pointer to location of accept status: */
  1145. xdr_ressize_check(rqstp, reply_statp);
  1146. svc_putnl(resv, 1); /* REJECT */
  1147. svc_putnl(resv, 1); /* AUTH_ERROR */
  1148. svc_putnl(resv, ntohl(auth_stat)); /* status */
  1149. goto sendit;
  1150. err_bad_prog:
  1151. dprintk("svc: unknown program %d\n", prog);
  1152. serv->sv_stats->rpcbadfmt++;
  1153. svc_putnl(resv, RPC_PROG_UNAVAIL);
  1154. goto sendit;
  1155. err_bad_vers:
  1156. svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
  1157. vers, prog, progp->pg_name);
  1158. serv->sv_stats->rpcbadfmt++;
  1159. svc_putnl(resv, RPC_PROG_MISMATCH);
  1160. svc_putnl(resv, progp->pg_lovers);
  1161. svc_putnl(resv, progp->pg_hivers);
  1162. goto sendit;
  1163. err_bad_proc:
  1164. svc_printk(rqstp, "unknown procedure (%d)\n", proc);
  1165. serv->sv_stats->rpcbadfmt++;
  1166. svc_putnl(resv, RPC_PROC_UNAVAIL);
  1167. goto sendit;
  1168. err_garbage:
  1169. svc_printk(rqstp, "failed to decode args\n");
  1170. rpc_stat = rpc_garbage_args;
  1171. err_bad:
  1172. serv->sv_stats->rpcbadfmt++;
  1173. svc_putnl(resv, ntohl(rpc_stat));
  1174. goto sendit;
  1175. }
  1176. /*
  1177. * Process the RPC request.
  1178. */
  1179. int
  1180. svc_process(struct svc_rqst *rqstp)
  1181. {
  1182. struct kvec *argv = &rqstp->rq_arg.head[0];
  1183. struct kvec *resv = &rqstp->rq_res.head[0];
  1184. struct svc_serv *serv = rqstp->rq_server;
  1185. u32 dir;
  1186. /*
  1187. * Setup response xdr_buf.
  1188. * Initially it has just one page
  1189. */
  1190. rqstp->rq_next_page = &rqstp->rq_respages[1];
  1191. resv->iov_base = page_address(rqstp->rq_respages[0]);
  1192. resv->iov_len = 0;
  1193. rqstp->rq_res.pages = rqstp->rq_respages + 1;
  1194. rqstp->rq_res.len = 0;
  1195. rqstp->rq_res.page_base = 0;
  1196. rqstp->rq_res.page_len = 0;
  1197. rqstp->rq_res.buflen = PAGE_SIZE;
  1198. rqstp->rq_res.tail[0].iov_base = NULL;
  1199. rqstp->rq_res.tail[0].iov_len = 0;
  1200. dir = svc_getnl(argv);
  1201. if (dir != 0) {
  1202. /* direction != CALL */
  1203. svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
  1204. serv->sv_stats->rpcbadfmt++;
  1205. goto out_drop;
  1206. }
  1207. /* Returns 1 for send, 0 for drop */
  1208. if (likely(svc_process_common(rqstp, argv, resv))) {
  1209. int ret = svc_send(rqstp);
  1210. trace_svc_process(rqstp, ret);
  1211. return ret;
  1212. }
  1213. out_drop:
  1214. trace_svc_process(rqstp, 0);
  1215. svc_drop(rqstp);
  1216. return 0;
  1217. }
  1218. EXPORT_SYMBOL_GPL(svc_process);
  1219. #if defined(CONFIG_SUNRPC_BACKCHANNEL)
  1220. /*
  1221. * Process a backchannel RPC request that arrived over an existing
  1222. * outbound connection
  1223. */
  1224. int
  1225. bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
  1226. struct svc_rqst *rqstp)
  1227. {
  1228. struct kvec *argv = &rqstp->rq_arg.head[0];
  1229. struct kvec *resv = &rqstp->rq_res.head[0];
  1230. struct rpc_task *task;
  1231. int proc_error;
  1232. int error;
  1233. dprintk("svc: %s(%p)\n", __func__, req);
  1234. /* Build the svc_rqst used by the common processing routine */
  1235. rqstp->rq_xid = req->rq_xid;
  1236. rqstp->rq_prot = req->rq_xprt->prot;
  1237. rqstp->rq_server = serv;
  1238. rqstp->rq_bc_net = req->rq_xprt->xprt_net;
  1239. rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
  1240. memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
  1241. memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
  1242. memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
  1243. /* Adjust the argument buffer length */
  1244. rqstp->rq_arg.len = req->rq_private_buf.len;
  1245. if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
  1246. rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
  1247. rqstp->rq_arg.page_len = 0;
  1248. } else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
  1249. rqstp->rq_arg.page_len)
  1250. rqstp->rq_arg.page_len = rqstp->rq_arg.len -
  1251. rqstp->rq_arg.head[0].iov_len;
  1252. else
  1253. rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
  1254. rqstp->rq_arg.page_len;
  1255. /* reset result send buffer "put" position */
  1256. resv->iov_len = 0;
  1257. /*
  1258. * Skip the next two words because they've already been
  1259. * processed in the transport
  1260. */
  1261. svc_getu32(argv); /* XID */
  1262. svc_getnl(argv); /* CALLDIR */
  1263. /* Parse and execute the bc call */
  1264. proc_error = svc_process_common(rqstp, argv, resv);
  1265. atomic_inc(&req->rq_xprt->bc_free_slots);
  1266. if (!proc_error) {
  1267. /* Processing error: drop the request */
  1268. xprt_free_bc_request(req);
  1269. return 0;
  1270. }
  1271. /* Finally, send the reply synchronously */
  1272. memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
  1273. task = rpc_run_bc_task(req);
  1274. if (IS_ERR(task)) {
  1275. error = PTR_ERR(task);
  1276. goto out;
  1277. }
  1278. WARN_ON_ONCE(atomic_read(&task->tk_count) != 1);
  1279. error = task->tk_status;
  1280. rpc_put_task(task);
  1281. out:
  1282. dprintk("svc: %s(), error=%d\n", __func__, error);
  1283. return error;
  1284. }
  1285. EXPORT_SYMBOL_GPL(bc_svc_process);
  1286. #endif /* CONFIG_SUNRPC_BACKCHANNEL */
  1287. /*
  1288. * Return (transport-specific) limit on the rpc payload.
  1289. */
  1290. u32 svc_max_payload(const struct svc_rqst *rqstp)
  1291. {
  1292. u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
  1293. if (rqstp->rq_server->sv_max_payload < max)
  1294. max = rqstp->rq_server->sv_max_payload;
  1295. return max;
  1296. }
  1297. EXPORT_SYMBOL_GPL(svc_max_payload);