1
2 #include <stdbool.h>
3 #include <stdint.h>
4
5 #include <sys/kconfig.h>
6 #include <sys/kassert.h>
7 #include <sys/kdebug.h>
8 #include <sys/kmem.h>
9
10 #include <machine/amd64.h>
11 #include <machine/amd64op.h>
12 #include <machine/mp.h>
13 #include <machine/pmap.h>
14
15 AS systemAS;
16 AS *currentAS[MAX_CPUS];
17
18 void
PMap_Init()19 PMap_Init()
20 {
21 int i, j;
22
23 kprintf("Initializing PMAP ... ");
24
25 // Setup global state
26 for (i = 0; i < MAX_CPUS; i++) {
27 currentAS[i] = 0;
28 }
29
30 // Allocate system page table
31 systemAS.root = PAlloc_AllocPage();
32 systemAS.tables = PAGETABLE_ENTRIES / 2 + 1;
33 systemAS.mappings = 0;
34 if (!systemAS.root)
35 PANIC("Cannot allocate system page table");
36
37 for (i = 0; i < PAGETABLE_ENTRIES / 2; i++)
38 systemAS.root->entries[i] = 0;
39
40 for (i = PAGETABLE_ENTRIES / 2; i < PAGETABLE_ENTRIES; i++) {
41 PageTable *pgtbl = PAlloc_AllocPage();
42 PageEntry pte = DMVA2PA((uint64_t)pgtbl) | PTE_W | PTE_P;
43 if (!pgtbl)
44 PANIC("Not enough memory!");
45
46 systemAS.root->entries[i] = pte;
47
48 for (j = 0; j < PAGETABLE_ENTRIES; j++) {
49 pgtbl->entries[j] = 0;
50 }
51 }
52
53 // Setup system mappings
54 PMap_SystemLMap(0x0, MEM_DIRECTMAP_BASE + 0x0,
55 3*512, 0); // 3GB RWX
56 PMap_SystemLMap(0xC0000000, MEM_DIRECTMAP_BASE + 0xC0000000,
57 512, PTE_NX|PTE_PCD); // 1GB RW + PCD
58 PMap_SystemLMap(0x100000000, MEM_DIRECTMAP_BASE + 0x100000000,
59 60*512, 0); // 60GB RWX
60
61 PMap_LoadAS(&systemAS);
62
63 kprintf("Done!\n");
64 }
65
66 void
PMap_InitAP()67 PMap_InitAP()
68 {
69 PMap_LoadAS(&systemAS);
70 }
71
72 /**
73 * PMap_NewAS --
74 *
75 * Create a new address space.
76 *
77 * @return Newly created address space.
78 */
79 AS*
PMap_NewAS()80 PMap_NewAS()
81 {
82 int i;
83 AS *as = PAlloc_AllocPage();
84
85 if (!as)
86 return NULL;
87
88 as->root = PAlloc_AllocPage();
89 as->tables = 1;
90 as->mappings = 0;
91
92 if (!as->root) {
93 PAlloc_Release(as);
94 return NULL;
95 }
96
97 for (i = 0; i < PAGETABLE_ENTRIES / 2; i++)
98 {
99 as->root->entries[i] = 0;
100 }
101 for (i = PAGETABLE_ENTRIES / 2; i < PAGETABLE_ENTRIES; i++) {
102 as->root->entries[i] = systemAS.root->entries[i];
103 }
104
105 return as;
106 }
107
108 /**
109 * PMap_DestroyAS --
110 *
111 * Destroys an address space and releases the physical pages.
112 *
113 * @param [in] space Address space to destroy.
114 */
115 void
PMap_DestroyAS(AS * space)116 PMap_DestroyAS(AS *space)
117 {
118 // Only free the userspace portion (bottom half)
119 for (int i = 0; i < PAGETABLE_ENTRIES / 2; i++)
120 {
121 PageEntry pte = space->root->entries[i];
122 if (pte & PTE_P) {
123 // Remove sub-pages
124 PageTable *tbl2 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
125 for (int j = 0; j < PAGETABLE_ENTRIES; j++) {
126 PageEntry pte2 = tbl2->entries[j];
127 if (pte2 & PTE_P) {
128 PageTable *tbl3 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
129 for (int k = 0; k < PAGETABLE_ENTRIES; k++) {
130 PageEntry pte3 = tbl3->entries[k];
131 if (pte3 & PTE_P) {
132 ASSERT((pte3 & PTE_PS) == 0); // XXX: Large pages not supported
133 PageTable *tbl4 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
134 for (int l = 0; l < PAGETABLE_ENTRIES; l++) {
135 PageEntry pte4 = tbl4->entries[l];
136 if (pte4 & PTE_P) {
137 // Free userspace page
138 PAlloc_Release((void *)DMPA2VA(pte4 & PGNUMMASK));
139 }
140 }
141
142 // Free 3rd level page table page
143 PAlloc_Release((void *)DMPA2VA(pte3 & PGNUMMASK));
144 }
145 }
146
147 // Free 2nd level page table page
148 PAlloc_Release((void *)DMPA2VA(pte2 & PGNUMMASK));
149 }
150 }
151
152 // Free page table page
153 PAlloc_Release((void *)DMPA2VA(pte & PGNUMMASK));
154 }
155 }
156
157 PAlloc_Release(space);
158 }
159
160 /**
161 * PMap_CurrentAS --
162 *
163 * Get the current address space on this CPU.
164 *
165 * @return Current address space.
166 */
167 AS *
PMap_CurrentAS()168 PMap_CurrentAS()
169 {
170 return currentAS[THISCPU()];
171 }
172
173 /**
174 * PMap_LoadAS --
175 *
176 * Load an address space into the CPU. Reloads the CR3 register in x86-64 that
177 * points the physical page tables and flushes the TLB entries.
178 *
179 * @param [in] space Address space to load.
180 */
181 void
PMap_LoadAS(AS * space)182 PMap_LoadAS(AS *space)
183 {
184 write_cr3(DMVA2PA((uint64_t)space->root));
185 currentAS[THISCPU()] = space;
186 }
187
188 /**
189 * PMapAllocPageTable --
190 *
191 * Allocates and initializes a page table.
192 *
193 * @return Newly created PageTable.
194 */
195 static PageTable *
PMapAllocPageTable()196 PMapAllocPageTable()
197 {
198 int i;
199 PageTable *pgtbl = PAlloc_AllocPage();
200
201 if (!pgtbl)
202 return 0;
203
204 for (i = 0; i < PAGETABLE_ENTRIES; i++) {
205 pgtbl->entries[i] = 0;
206 }
207
208 return pgtbl;
209 }
210
211 /**
212 * PMap_Translate --
213 *
214 * Translates a virtual address to physical address for a given address space.
215 *
216 * @param [in] space Address space we wish to lookup a mapping in.
217 * @param [in] va Virtual address we wish to translate.
218 */
219 uintptr_t
PMap_Translate(AS * space,uintptr_t va)220 PMap_Translate(AS *space, uintptr_t va)
221 {
222 int i,j,k,l;
223 PageTable *table = space->root;
224 PageEntry pte;
225 PageEntry *entry;
226
227 i = (va >> (HUGE_PGSHIFT + PGIDXSHIFT)) & PGIDXMASK;
228 j = (va >> HUGE_PGSHIFT) & PGIDXMASK;
229 k = (va >> LARGE_PGSHIFT) & PGIDXMASK;
230 l = (va >> PGSHIFT) & PGIDXMASK;
231
232 pte = table->entries[i];
233 if (pte == 0) {
234 ASSERT(pte);
235 return 0;
236 }
237 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
238
239 pte = table->entries[j];
240 // XXX: Support 1GB pages
241 if (pte == 0) {
242 ASSERT(pte);
243 return 0;
244 }
245 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
246
247 pte = table->entries[k];
248 if ((pte & PTE_PS) == PTE_PS) {
249 // Handle 2MB pages
250 entry = &table->entries[k];
251 return (*entry & ~(LARGE_PGMASK | PTE_NX)) + (va & LARGE_PGMASK);
252 }
253 if (pte == 0) {
254 ASSERT(pte);
255 return 0;
256 }
257 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
258
259 // Handle 4KB pages
260 entry = &table->entries[l];
261
262 return (*entry & ~(PGMASK | PTE_NX)) + (va & PGMASK);
263 }
264
265 /**
266 * PMapLookupEntry --
267 *
268 * Lookup a virtual address in a page table and return a pointer to the page
269 * entry. This function allocates page tables as necessary to fill in the
270 * 4-level heirarchy.
271 *
272 * @param [in] space Address space to search.
273 * @param [in] va Virtual address to lookup.
274 * @param [out] entry Pointer will point to the PageEntry.
275 * @param [in] size Page size we want to use.
276 */
277 static void
PMapLookupEntry(AS * space,uint64_t va,PageEntry ** entry,int size)278 PMapLookupEntry(AS *space, uint64_t va, PageEntry **entry, int size)
279 {
280 int i,j,k,l;
281 PageTable *table = space->root;
282 PageEntry pte;
283
284 i = (va >> (HUGE_PGSHIFT + PGIDXSHIFT)) & PGIDXMASK;
285 j = (va >> HUGE_PGSHIFT) & PGIDXMASK;
286 k = (va >> LARGE_PGSHIFT) & PGIDXMASK;
287 l = (va >> PGSHIFT) & PGIDXMASK;
288
289 *entry = NULL;
290
291 pte = table->entries[i];
292 if (pte == 0) {
293 PageTable *newtable = PMapAllocPageTable();
294 if (!newtable)
295 return;
296
297 pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
298 table->entries[i] = pte;
299 }
300 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
301
302 pte = table->entries[j];
303 if (size == HUGE_PGSIZE) {
304 // Handle 1GB pages
305 *entry = &table->entries[j];
306 return;
307 }
308 if (pte == 0) {
309 PageTable *newtable = PMapAllocPageTable();
310 if (!newtable)
311 return;
312
313 pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
314 table->entries[j] = pte;
315 }
316 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
317
318 pte = table->entries[k];
319 if (size == LARGE_PGSIZE) {
320 // Handle 2MB pages
321 *entry = &table->entries[k];
322 return;
323 }
324 if (pte == 0) {
325 PageTable *newtable = PMapAllocPageTable();
326 if (!newtable)
327 return;
328
329 pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
330 table->entries[k] = pte;
331 }
332 table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
333
334 // Handle 4KB pages
335 ASSERT(size == PGSIZE);
336 *entry = &table->entries[l];
337 return;
338 }
339
340 /**
341 * PMap_Map --
342 *
343 * Map a physical to virtual mapping in an address space.
344 *
345 * @param [in] as Address space.
346 * @param [in] phys Physical address.
347 * @param [in] virt Virtual address.
348 * @param [in] pages Pages to map in.
349 * @param [in] flags Flags to apply to the mapping.
350 *
351 * @retval true On success
352 * @retval false On failure
353 */
354 bool
PMap_Map(AS * as,uint64_t phys,uint64_t virt,uint64_t pages,uint64_t flags)355 PMap_Map(AS *as, uint64_t phys, uint64_t virt, uint64_t pages, uint64_t flags)
356 {
357 int i;
358 PageEntry *entry;
359
360 for (i = 0; i < pages; i++) {
361 uint64_t va = virt + PGSIZE * i;
362 PMapLookupEntry(as, va, &entry, PGSIZE);
363 if (!entry) {
364 kprintf("Map failed to allocate memory!\n");
365 return false;
366 }
367
368 *entry = (phys + PGSIZE * i) | PTE_P | PTE_W | PTE_U | flags;
369 }
370
371 return true;
372 }
373
374 /**
375 * PMap_Unmap --
376 *
377 * Unmap a range of addresses.
378 *
379 * @param [in] as Address space.
380 * @param [in] va Virtual address.
381 * @param [in] pages Pages to map in.
382 *
383 * @retval true On success
384 * @retval false On failure
385 */
386 bool
PMap_Unmap(AS * as,uint64_t va,uint64_t pages)387 PMap_Unmap(AS *as, uint64_t va, uint64_t pages)
388 {
389 int i;
390 PageEntry *entry;
391
392 for (i = 0; i < pages; i++) {
393 uint64_t vai = va + PGSIZE * i;
394 PMapLookupEntry(as, vai, &entry, PGSIZE);
395 if (!entry) {
396 kprintf("Unmap tried to allocate memory!\n");
397 return false;
398 }
399
400 NOT_IMPLEMENTED();
401
402 *entry = 0;
403 }
404
405 return true;
406 }
407
408 /**
409 * PMap_AllocMap --
410 *
411 * Map a virtual mapping in an address space and back it by newly allocated
412 * memory.
413 *
414 * @param [in] as Address space.
415 * @param [in] virt Virtual address.
416 * @param [in] pages Pages to map in.
417 * @param [in] flags Flags to apply to the mapping.
418 *
419 * @retval true On success
420 * @retval false On failure
421 */
422 bool
PMap_AllocMap(AS * as,uint64_t virt,uint64_t len,uint64_t flags)423 PMap_AllocMap(AS *as, uint64_t virt, uint64_t len, uint64_t flags)
424 {
425 int i;
426 uint64_t pages = (len + PGSIZE - 1) / PGSIZE;
427 PageEntry *entry;
428
429 ASSERT((virt & PGMASK) == 0);
430
431 for (i = 0; i < pages; i++) {
432 uint64_t va = virt + PGSIZE * i;
433 PMapLookupEntry(as, va, &entry, PGSIZE);
434 if (!entry) {
435 kprintf("Map failed to allocate memory!\n");
436 return false;
437 }
438
439 if ((*entry & PTE_P) != PTE_P) {
440 void *pg = PAlloc_AllocPage();
441 *entry = (uint64_t)DMVA2PA(pg) | PTE_P | PTE_U | flags;
442 }
443 }
444
445 return true;
446 }
447
448 /**
449 * PMap_SystemLookup --
450 *
451 * Lookup a kernel virtual address in a page table and return a pointer to the
452 * page entry. This function allocates page tables as necessary to fill in the
453 * 4-level heirarchy.
454 *
455 * @param [in] va Virtual address to lookup.
456 * @param [out] entry Pointer will point to the PageEntry.
457 * @param [in] size Page size we want to use.
458 */
459 void
PMap_SystemLookup(uint64_t va,PageEntry ** entry,int size)460 PMap_SystemLookup(uint64_t va, PageEntry **entry, int size)
461 {
462 PMapLookupEntry(&systemAS, va, entry, size);
463 }
464
465 /**
466 * PMap_SystemLMap --
467 *
468 * Map a range of large (2MB) physical pages to virtual pages in the kernel
469 * address space that is shared by all processes.
470 *
471 * @param [in] phys Physical address.
472 * @param [in] virt Virtual address.
473 * @param [in] lpages Large pages to map in.
474 * @param [in] flags Flags to apply to the mapping.
475 *
476 * @retval true On success
477 * @retval false On failure
478 */
479 bool
PMap_SystemLMap(uint64_t phys,uint64_t virt,uint64_t lpages,uint64_t flags)480 PMap_SystemLMap(uint64_t phys, uint64_t virt, uint64_t lpages, uint64_t flags)
481 {
482 int i;
483 PageEntry *entry;
484
485 for (i = 0; i < lpages; i++) {
486 uint64_t va = virt + LARGE_PGSIZE * i;
487 PMapLookupEntry(&systemAS, va, &entry, LARGE_PGSIZE);
488 if (!entry) {
489 kprintf("SystemLMap failed to allocate memory!\n");
490 return false;
491 }
492
493 *entry = (phys + LARGE_PGSIZE * i) | PTE_P | PTE_W | PTE_PS | flags;
494 }
495
496 return true;
497 }
498
499 /**
500 * PMap_SystemLMap --
501 *
502 * Map a range of physical pages to virtual pages in the kernel address space
503 * that is shared by all processes.
504 *
505 * @param [in] phys Physical address.
506 * @param [in] virt Virtual address.
507 * @param [in] pages Pages to map in.
508 * @param [in] flags Flags to apply to the mapping.
509 *
510 * @retval true On success
511 * @retval false On failure
512 */
513 bool
PMap_SystemMap(uint64_t phys,uint64_t virt,uint64_t pages,uint64_t flags)514 PMap_SystemMap(uint64_t phys, uint64_t virt, uint64_t pages, uint64_t flags)
515 {
516 int i;
517 PageEntry *entry;
518
519 for (i = 0; i < pages; i++) {
520 uint64_t va = virt + PGSIZE * i;
521 PMapLookupEntry(&systemAS, va, &entry, PGSIZE);
522 if (!entry) {
523 kprintf("SystemMap failed to allocate memory!\n");
524 return false;
525 }
526
527 *entry = (phys + PGSIZE * i) | PTE_P | PTE_W | flags;
528 }
529
530 return true;
531 }
532
533 /**
534 * PMap_SystemUnmap --
535 *
536 * We do not currently use this!
537 */
538 bool
PMap_SystemUnmap(uint64_t virt,uint64_t pages)539 PMap_SystemUnmap(uint64_t virt, uint64_t pages)
540 {
541 NOT_IMPLEMENTED();
542 return false;
543 }
544
545 static uint64_t
AddrFromIJKL(uint64_t i,uint64_t j,uint64_t k,uint64_t l)546 AddrFromIJKL(uint64_t i, uint64_t j, uint64_t k, uint64_t l)
547 {
548 return (i << 39) | (j << HUGE_PGSHIFT) | (k << LARGE_PGSHIFT) | (l << PGSHIFT);
549 }
550
551 void
PMap_DumpFull(AS * space)552 PMap_DumpFull(AS *space)
553 {
554 int i = 0;
555 int j = 0;
556 int k = 0;
557 int l = 0;
558 PageTable *root = space->root;
559
560 kprintf("Root: %016llx\n", (uint64_t)space->root);
561
562 for (i = 0; i < PAGETABLE_ENTRIES; i++) {
563 PageEntry pte = root->entries[i];
564 PageTable *l1 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
565
566 if (!(pte & PTE_P))
567 continue;
568
569 kprintf("Level 1: %016llx\n", (uint64_t)pte);
570
571 for (j = 0; j < PAGETABLE_ENTRIES; j++) {
572 PageEntry pte2 = l1->entries[j];
573 PageTable *l2 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
574
575 if (!(pte2 & PTE_P))
576 continue;
577
578 kprintf("Level 2: %016llx\n", (uint64_t)pte2);
579
580 for (k = 0; k < PAGETABLE_ENTRIES; k++) {
581 PageEntry pte3 = l2->entries[k];
582 PageTable *l3 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
583
584 if (!(pte3 & PTE_P))
585 continue;
586
587 kprintf("Level 3: %016llx:%016llx\n",
588 AddrFromIJKL(i, j, k, 0),
589 (uint64_t)pte3);
590
591 if ((pte3 & PTE_PS) == 0) {
592 for (l = 0; l < PAGETABLE_ENTRIES; l++) {
593 PageEntry pte4 = l3->entries[l];
594
595 kprintf("Level 4: %016llx:%016llx\n",
596 AddrFromIJKL(i, j, k, l),
597 (uint64_t)pte4);
598 }
599 }
600 }
601 }
602 }
603
604 return;
605 }
606
607 static void
Debug_PMapDumpFull(int argc,const char * argv[])608 Debug_PMapDumpFull(int argc, const char *argv[])
609 {
610 PMap_DumpFull(currentAS[THISCPU()]);
611 }
612
613 REGISTER_DBGCMD(pmapdumpfull, "Dump memory mappings", Debug_PMapDumpFull);
614
615 void
PMap_Dump(AS * space)616 PMap_Dump(AS *space)
617 {
618 int i = 0;
619 int j = 0;
620 int k = 0;
621 int l = 0;
622 PageTable *root = space->root;
623
624 kprintf("%-18s %-18s %-5s\n", "Virtual", "Physical PTE", "Flags");
625 for (i = 0; i < PAGETABLE_ENTRIES/2; i++) {
626 PageEntry pte = root->entries[i];
627 PageTable *l1 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
628
629 if (!(pte & PTE_P))
630 continue;
631
632 for (j = 0; j < PAGETABLE_ENTRIES; j++) {
633 PageEntry pte2 = l1->entries[j];
634 PageTable *l2 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
635
636 if (!(pte2 & PTE_P))
637 continue;
638
639 for (k = 0; k < PAGETABLE_ENTRIES; k++) {
640 PageEntry pte3 = l2->entries[k];
641 PageTable *l3 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
642
643 if (!(pte3 & PTE_P))
644 continue;
645
646 if ((pte3 & PTE_PS) == 0) {
647 for (l = 0; l < PAGETABLE_ENTRIES; l++) {
648 PageEntry pte4 = l3->entries[l];
649
650 if (pte4 & PTE_P)
651 kprintf("0x%016llx: 0x%016llx P%c%c%c%c%c\n",
652 AddrFromIJKL(i, j, k, l),
653 (uint64_t)pte4,
654 (pte4 & PTE_W) ? 'W' : ' ',
655 (pte4 & PTE_NX) ? ' ' : 'X',
656 (pte4 & PTE_U) ? 'U' : ' ',
657 (pte4 & PTE_A) ? 'A' : ' ',
658 (pte4 & PTE_D) ? 'D' : ' ');
659 }
660 }
661 }
662 }
663 }
664
665 return;
666 }
667
668 static void
Debug_PMapDump(int argc,const char * argv[])669 Debug_PMapDump(int argc, const char *argv[])
670 {
671 PMap_Dump(currentAS[THISCPU()]);
672 }
673
674 REGISTER_DBGCMD(pmapdump, "Dump memory mappings", Debug_PMapDump);
675
676
677