tsb.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. #ifndef _SPARC64_TSB_H
  2. #define _SPARC64_TSB_H
  3. /* The sparc64 TSB is similar to the powerpc hashtables. It's a
  4. * power-of-2 sized table of TAG/PTE pairs. The cpu precomputes
  5. * pointers into this table for 8K and 64K page sizes, and also a
  6. * comparison TAG based upon the virtual address and context which
  7. * faults.
  8. *
  9. * TLB miss trap handler software does the actual lookup via something
  10. * of the form:
  11. *
  12. * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
  13. * ldxa [%g0] ASI_{D,I}MMU, %g6
  14. * sllx %g6, 22, %g6
  15. * srlx %g6, 22, %g6
  16. * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4
  17. * cmp %g4, %g6
  18. * bne,pn %xcc, tsb_miss_{d,i}tlb
  19. * mov FAULT_CODE_{D,I}TLB, %g3
  20. * stxa %g5, [%g0] ASI_{D,I}TLB_DATA_IN
  21. * retry
  22. *
  23. *
  24. * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
  25. * PTE. The TAG is of the same layout as the TLB TAG TARGET mmu
  26. * register which is:
  27. *
  28. * -------------------------------------------------
  29. * | - | CONTEXT | - | VADDR bits 63:22 |
  30. * -------------------------------------------------
  31. * 63 61 60 48 47 42 41 0
  32. *
  33. * But actually, since we use per-mm TSB's, we zero out the CONTEXT
  34. * field.
  35. *
  36. * Like the powerpc hashtables we need to use locking in order to
  37. * synchronize while we update the entries. PTE updates need locking
  38. * as well.
  39. *
  40. * We need to carefully choose a lock bits for the TSB entry. We
  41. * choose to use bit 47 in the tag. Also, since we never map anything
  42. * at page zero in context zero, we use zero as an invalid tag entry.
  43. * When the lock bit is set, this forces a tag comparison failure.
  44. */
  45. #define TSB_TAG_LOCK_BIT 47
  46. #define TSB_TAG_LOCK_HIGH (1 << (TSB_TAG_LOCK_BIT - 32))
  47. #define TSB_TAG_INVALID_BIT 46
  48. #define TSB_TAG_INVALID_HIGH (1 << (TSB_TAG_INVALID_BIT - 32))
  49. /* Some cpus support physical address quad loads. We want to use
  50. * those if possible so we don't need to hard-lock the TSB mapping
  51. * into the TLB. We encode some instruction patching in order to
  52. * support this.
  53. *
  54. * The kernel TSB is locked into the TLB by virtue of being in the
  55. * kernel image, so we don't play these games for swapper_tsb access.
  56. */
  57. #ifndef __ASSEMBLY__
  58. struct tsb_ldquad_phys_patch_entry {
  59. unsigned int addr;
  60. unsigned int sun4u_insn;
  61. unsigned int sun4v_insn;
  62. };
  63. extern struct tsb_ldquad_phys_patch_entry __tsb_ldquad_phys_patch,
  64. __tsb_ldquad_phys_patch_end;
  65. struct tsb_phys_patch_entry {
  66. unsigned int addr;
  67. unsigned int insn;
  68. };
  69. extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
  70. #endif
  71. #define TSB_LOAD_QUAD(TSB, REG) \
  72. 661: ldda [TSB] ASI_NUCLEUS_QUAD_LDD, REG; \
  73. .section .tsb_ldquad_phys_patch, "ax"; \
  74. .word 661b; \
  75. ldda [TSB] ASI_QUAD_LDD_PHYS, REG; \
  76. ldda [TSB] ASI_QUAD_LDD_PHYS_4V, REG; \
  77. .previous
  78. #define TSB_LOAD_TAG_HIGH(TSB, REG) \
  79. 661: lduwa [TSB] ASI_N, REG; \
  80. .section .tsb_phys_patch, "ax"; \
  81. .word 661b; \
  82. lduwa [TSB] ASI_PHYS_USE_EC, REG; \
  83. .previous
  84. #define TSB_LOAD_TAG(TSB, REG) \
  85. 661: ldxa [TSB] ASI_N, REG; \
  86. .section .tsb_phys_patch, "ax"; \
  87. .word 661b; \
  88. ldxa [TSB] ASI_PHYS_USE_EC, REG; \
  89. .previous
  90. #define TSB_CAS_TAG_HIGH(TSB, REG1, REG2) \
  91. 661: casa [TSB] ASI_N, REG1, REG2; \
  92. .section .tsb_phys_patch, "ax"; \
  93. .word 661b; \
  94. casa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \
  95. .previous
  96. #define TSB_CAS_TAG(TSB, REG1, REG2) \
  97. 661: casxa [TSB] ASI_N, REG1, REG2; \
  98. .section .tsb_phys_patch, "ax"; \
  99. .word 661b; \
  100. casxa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \
  101. .previous
  102. #define TSB_STORE(ADDR, VAL) \
  103. 661: stxa VAL, [ADDR] ASI_N; \
  104. .section .tsb_phys_patch, "ax"; \
  105. .word 661b; \
  106. stxa VAL, [ADDR] ASI_PHYS_USE_EC; \
  107. .previous
  108. #define TSB_LOCK_TAG(TSB, REG1, REG2) \
  109. 99: TSB_LOAD_TAG_HIGH(TSB, REG1); \
  110. sethi %hi(TSB_TAG_LOCK_HIGH), REG2;\
  111. andcc REG1, REG2, %g0; \
  112. bne,pn %icc, 99b; \
  113. nop; \
  114. TSB_CAS_TAG_HIGH(TSB, REG1, REG2); \
  115. cmp REG1, REG2; \
  116. bne,pn %icc, 99b; \
  117. nop; \
  118. #define TSB_WRITE(TSB, TTE, TAG) \
  119. add TSB, 0x8, TSB; \
  120. TSB_STORE(TSB, TTE); \
  121. sub TSB, 0x8, TSB; \
  122. TSB_STORE(TSB, TAG);
  123. /* Do a kernel page table walk. Leaves valid PTE value in
  124. * REG1. Jumps to FAIL_LABEL on early page table walk
  125. * termination. VADDR will not be clobbered, but REG2 will.
  126. *
  127. * There are two masks we must apply to propagate bits from
  128. * the virtual address into the PTE physical address field
  129. * when dealing with huge pages. This is because the page
  130. * table boundaries do not match the huge page size(s) the
  131. * hardware supports.
  132. *
  133. * In these cases we propagate the bits that are below the
  134. * page table level where we saw the huge page mapping, but
  135. * are still within the relevant physical bits for the huge
  136. * page size in question. So for PMD mappings (which fall on
  137. * bit 23, for 8MB per PMD) we must propagate bit 22 for a
  138. * 4MB huge page. For huge PUDs (which fall on bit 33, for
  139. * 8GB per PUD), we have to accommodate 256MB and 2GB huge
  140. * pages. So for those we propagate bits 32 to 28.
  141. */
  142. #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
  143. sethi %hi(swapper_pg_dir), REG1; \
  144. or REG1, %lo(swapper_pg_dir), REG1; \
  145. sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
  146. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  147. andn REG2, 0x7, REG2; \
  148. ldx [REG1 + REG2], REG1; \
  149. brz,pn REG1, FAIL_LABEL; \
  150. sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
  151. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  152. andn REG2, 0x7, REG2; \
  153. ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
  154. brz,pn REG1, FAIL_LABEL; \
  155. sethi %uhi(_PAGE_PUD_HUGE), REG2; \
  156. brz,pn REG1, FAIL_LABEL; \
  157. sllx REG2, 32, REG2; \
  158. andcc REG1, REG2, %g0; \
  159. sethi %hi(0xf8000000), REG2; \
  160. bne,pt %xcc, 697f; \
  161. sllx REG2, 1, REG2; \
  162. sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
  163. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  164. andn REG2, 0x7, REG2; \
  165. ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
  166. sethi %uhi(_PAGE_PMD_HUGE), REG2; \
  167. brz,pn REG1, FAIL_LABEL; \
  168. sllx REG2, 32, REG2; \
  169. andcc REG1, REG2, %g0; \
  170. be,pn %xcc, 698f; \
  171. sethi %hi(0x400000), REG2; \
  172. 697: brgez,pn REG1, FAIL_LABEL; \
  173. andn REG1, REG2, REG1; \
  174. and VADDR, REG2, REG2; \
  175. ba,pt %xcc, 699f; \
  176. or REG1, REG2, REG1; \
  177. 698: sllx VADDR, 64 - PMD_SHIFT, REG2; \
  178. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  179. andn REG2, 0x7, REG2; \
  180. ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
  181. brgez,pn REG1, FAIL_LABEL; \
  182. nop; \
  183. 699:
  184. /* PMD has been loaded into REG1, interpret the value, seeing
  185. * if it is a HUGE PMD or a normal one. If it is not valid
  186. * then jump to FAIL_LABEL. If it is a HUGE PMD, and it
  187. * translates to a valid PTE, branch to PTE_LABEL.
  188. *
  189. * We have to propagate the 4MB bit of the virtual address
  190. * because we are fabricating 8MB pages using 4MB hw pages.
  191. */
  192. #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
  193. #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
  194. brz,pn REG1, FAIL_LABEL; \
  195. sethi %uhi(_PAGE_PMD_HUGE), REG2; \
  196. sllx REG2, 32, REG2; \
  197. andcc REG1, REG2, %g0; \
  198. be,pt %xcc, 700f; \
  199. sethi %hi(4 * 1024 * 1024), REG2; \
  200. brgez,pn REG1, FAIL_LABEL; \
  201. andn REG1, REG2, REG1; \
  202. and VADDR, REG2, REG2; \
  203. brlz,pt REG1, PTE_LABEL; \
  204. or REG1, REG2, REG1; \
  205. 700:
  206. #else
  207. #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
  208. brz,pn REG1, FAIL_LABEL; \
  209. nop;
  210. #endif
  211. /* Do a user page table walk in MMU globals. Leaves final,
  212. * valid, PTE value in REG1. Jumps to FAIL_LABEL on early
  213. * page table walk termination or if the PTE is not valid.
  214. *
  215. * Physical base of page tables is in PHYS_PGD which will not
  216. * be modified.
  217. *
  218. * VADDR will not be clobbered, but REG1 and REG2 will.
  219. */
  220. #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \
  221. sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
  222. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  223. andn REG2, 0x7, REG2; \
  224. ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
  225. brz,pn REG1, FAIL_LABEL; \
  226. sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
  227. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  228. andn REG2, 0x7, REG2; \
  229. ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
  230. brz,pn REG1, FAIL_LABEL; \
  231. sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
  232. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  233. andn REG2, 0x7, REG2; \
  234. ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
  235. USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
  236. sllx VADDR, 64 - PMD_SHIFT, REG2; \
  237. srlx REG2, 64 - PAGE_SHIFT, REG2; \
  238. andn REG2, 0x7, REG2; \
  239. add REG1, REG2, REG1; \
  240. ldxa [REG1] ASI_PHYS_USE_EC, REG1; \
  241. brgez,pn REG1, FAIL_LABEL; \
  242. nop; \
  243. 800:
  244. /* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
  245. * If no entry is found, FAIL_LABEL will be branched to. On success
  246. * the resulting PTE value will be left in REG1. VADDR is preserved
  247. * by this routine.
  248. */
  249. #define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
  250. sethi %hi(prom_trans), REG1; \
  251. or REG1, %lo(prom_trans), REG1; \
  252. 97: ldx [REG1 + 0x00], REG2; \
  253. brz,pn REG2, FAIL_LABEL; \
  254. nop; \
  255. ldx [REG1 + 0x08], REG3; \
  256. add REG2, REG3, REG3; \
  257. cmp REG2, VADDR; \
  258. bgu,pt %xcc, 98f; \
  259. cmp VADDR, REG3; \
  260. bgeu,pt %xcc, 98f; \
  261. ldx [REG1 + 0x10], REG3; \
  262. sub VADDR, REG2, REG2; \
  263. ba,pt %xcc, 99f; \
  264. add REG3, REG2, REG1; \
  265. 98: ba,pt %xcc, 97b; \
  266. add REG1, (3 * 8), REG1; \
  267. 99:
  268. /* We use a 32K TSB for the whole kernel, this allows to
  269. * handle about 16MB of modules and vmalloc mappings without
  270. * incurring many hash conflicts.
  271. */
  272. #define KERNEL_TSB_SIZE_BYTES (32 * 1024)
  273. #define KERNEL_TSB_NENTRIES \
  274. (KERNEL_TSB_SIZE_BYTES / 16)
  275. #define KERNEL_TSB4M_NENTRIES 4096
  276. /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
  277. * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
  278. * and the found TTE will be left in REG1. REG3 and REG4 must
  279. * be an even/odd pair of registers.
  280. *
  281. * VADDR and TAG will be preserved and not clobbered by this macro.
  282. */
  283. #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
  284. 661: sethi %uhi(swapper_tsb), REG1; \
  285. sethi %hi(swapper_tsb), REG2; \
  286. or REG1, %ulo(swapper_tsb), REG1; \
  287. or REG2, %lo(swapper_tsb), REG2; \
  288. .section .swapper_tsb_phys_patch, "ax"; \
  289. .word 661b; \
  290. .previous; \
  291. sllx REG1, 32, REG1; \
  292. or REG1, REG2, REG1; \
  293. srlx VADDR, PAGE_SHIFT, REG2; \
  294. and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
  295. sllx REG2, 4, REG2; \
  296. add REG1, REG2, REG2; \
  297. TSB_LOAD_QUAD(REG2, REG3); \
  298. cmp REG3, TAG; \
  299. be,a,pt %xcc, OK_LABEL; \
  300. mov REG4, REG1;
  301. #ifndef CONFIG_DEBUG_PAGEALLOC
  302. /* This version uses a trick, the TAG is already (VADDR >> 22) so
  303. * we can make use of that for the index computation.
  304. */
  305. #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
  306. 661: sethi %uhi(swapper_4m_tsb), REG1; \
  307. sethi %hi(swapper_4m_tsb), REG2; \
  308. or REG1, %ulo(swapper_4m_tsb), REG1; \
  309. or REG2, %lo(swapper_4m_tsb), REG2; \
  310. .section .swapper_4m_tsb_phys_patch, "ax"; \
  311. .word 661b; \
  312. .previous; \
  313. sllx REG1, 32, REG1; \
  314. or REG1, REG2, REG1; \
  315. and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
  316. sllx REG2, 4, REG2; \
  317. add REG1, REG2, REG2; \
  318. TSB_LOAD_QUAD(REG2, REG3); \
  319. cmp REG3, TAG; \
  320. be,a,pt %xcc, OK_LABEL; \
  321. mov REG4, REG1;
  322. #endif
  323. #endif /* !(_SPARC64_TSB_H) */