pmap.c File Reference

#include <stdbool.h>
#include <stdint.h>
#include <sys/kconfig.h>
#include <sys/kassert.h>
#include <sys/kdebug.h>
#include <sys/kmem.h>
#include <machine/amd64.h>
#include <machine/amd64op.h>
#include <machine/mp.h>
#include <machine/pmap.h>

Include dependency graph for pmap.c:

Go to the source code of this file.

Functions
void	PMap_Init ()
void	PMap_InitAP ()
AS *	PMap_NewAS ()
void	PMap_DestroyAS (AS *space)
AS *	PMap_CurrentAS ()
void	PMap_LoadAS (AS *space)
static PageTable *	PMapAllocPageTable ()
uintptr_t	PMap_Translate (AS *space, uintptr_t va)
static void	PMapLookupEntry (AS *space, uint64_t va, PageEntry **entry, int size)
bool	PMap_Map (AS *as, uint64_t phys, uint64_t virt, uint64_t pages, uint64_t flags)
bool	PMap_Unmap (AS *as, uint64_t va, uint64_t pages)
bool	PMap_AllocMap (AS *as, uint64_t virt, uint64_t len, uint64_t flags)
void	PMap_SystemLookup (uint64_t va, PageEntry **entry, int size)
bool	PMap_SystemLMap (uint64_t phys, uint64_t virt, uint64_t lpages, uint64_t flags)
bool	PMap_SystemMap (uint64_t phys, uint64_t virt, uint64_t pages, uint64_t flags)
bool	PMap_SystemUnmap (uint64_t virt, uint64_t pages)
static uint64_t	AddrFromIJKL (uint64_t i, uint64_t j, uint64_t k, uint64_t l)
void	PMap_DumpFull (AS *space)
static void	Debug_PMapDumpFull (int argc, const char *argv[])
	REGISTER_DBGCMD (pmapdumpfull, "Dump memory mappings", Debug_PMapDumpFull)
void	PMap_Dump (AS *space)
static void	Debug_PMapDump (int argc, const char *argv[])
	REGISTER_DBGCMD (pmapdump, "Dump memory mappings", Debug_PMapDump)

Variables
AS	systemAS
AS *	currentAS [MAX_CPUS]

Function Documentation

AddrFromIJKL()

static uint64_t AddrFromIJKL ( uint64_t  i, uint64_t  j, uint64_t  k, uint64_t  l  )

static

Definition at line 546 of file pmap.c.

{
    return (i << 39) | (j << HUGE_PGSHIFT) | (k << LARGE_PGSHIFT) | (l << PGSHIFT);
}

Here is the caller graph for this function:

Debug_PMapDump()

static void Debug_PMapDump ( int  argc, const char *  argv[]  )

static

Definition at line 669 of file pmap.c.

{
    PMap_Dump(currentAS[THISCPU()]);
}

Here is the call graph for this function:

Debug_PMapDumpFull()

static void Debug_PMapDumpFull ( int  argc, const char *  argv[]  )

static

Definition at line 608 of file pmap.c.

{
    PMap_DumpFull(currentAS[THISCPU()]);
}

Here is the call graph for this function:

PMap_AllocMap()

bool PMap_AllocMap	(	AS *	as,
		uint64_t	virt,
		uint64_t	len,
		uint64_t	flags
	)

PMap_AllocMap –

Map a virtual mapping in an address space and back it by newly allocated memory.

Parameters

[in]	as	Address space.
[in]	virt	Virtual address.
[in]	pages	Pages to map in.
[in]	flags	Flags to apply to the mapping.

Return values

true	On success
false	On failure

Definition at line 423 of file pmap.c.

{
    int i;
    uint64_t pages = (len + PGSIZE - 1) / PGSIZE;
    PageEntry *entry;
 
    ASSERT((virt & PGMASK) == 0);
 
    for (i = 0; i < pages; i++) {
        uint64_t va = virt + PGSIZE * i;
        PMapLookupEntry(as, va, &entry, PGSIZE);
        if (!entry) {
            kprintf("Map failed to allocate memory!\n");
            return false;
        }
 
        if ((*entry & PTE_P) != PTE_P) {
            void *pg = PAlloc_AllocPage();
            *entry = (uint64_t)DMVA2PA(pg) | PTE_P | PTE_U | flags;
        }
    }
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_CurrentAS()

AS * PMap_CurrentAS

(

)

PMap_CurrentAS –

Get the current address space on this CPU.

Returns

Current address space.

Definition at line 168 of file pmap.c.

{
    return currentAS[THISCPU()];
}

PMap_DestroyAS()

void PMap_DestroyAS

(

AS *

space

)

PMap_DestroyAS –

Destroys an address space and releases the physical pages.

Parameters

[in]

space

Address space to destroy.

Definition at line 116 of file pmap.c.

{
    // Only free the userspace portion (bottom half)
    for (int i = 0; i < PAGETABLE_ENTRIES / 2; i++)
    {
        PageEntry pte = space->root->entries[i];
        if (pte & PTE_P) {
            // Remove sub-pages
            PageTable *tbl2 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
            for (int j = 0; j < PAGETABLE_ENTRIES; j++) {
                PageEntry pte2 = tbl2->entries[j];
                if (pte2 & PTE_P) {
                    PageTable *tbl3 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
                    for (int k = 0; k < PAGETABLE_ENTRIES; k++) {
                        PageEntry pte3 = tbl3->entries[k];
                        if (pte3 & PTE_P) {
                            ASSERT((pte3 & PTE_PS) == 0); // XXX: Large pages not supported
                            PageTable *tbl4 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
                            for (int l = 0; l < PAGETABLE_ENTRIES; l++) {
                                PageEntry pte4 = tbl4->entries[l];
                                if (pte4 & PTE_P) {
                                    // Free userspace page
                                    PAlloc_Release((void *)DMPA2VA(pte4 & PGNUMMASK));
                                }
                            }
 
                            // Free 3rd level page table page
                            PAlloc_Release((void *)DMPA2VA(pte3 & PGNUMMASK));
                        }
                    }
 
                    // Free 2nd level page table page
                    PAlloc_Release((void *)DMPA2VA(pte2 & PGNUMMASK));
                }
            }
 
            // Free page table page
            PAlloc_Release((void *)DMPA2VA(pte & PGNUMMASK));
        }
    }
 
    PAlloc_Release(space);
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_Dump()

void PMap_Dump

(

AS *

space

)

Definition at line 616 of file pmap.c.

{
    int i = 0;
    int j = 0;
    int k = 0;
    int l = 0;
    PageTable *root = space->root;
 
    kprintf("%-18s  %-18s %-5s\n", "Virtual", "Physical PTE", "Flags");
    for (i = 0; i < PAGETABLE_ENTRIES/2; i++) {
        PageEntry pte = root->entries[i];
        PageTable *l1 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
        if (!(pte & PTE_P))
            continue;
 
        for (j = 0; j < PAGETABLE_ENTRIES; j++) {
            PageEntry pte2 = l1->entries[j];
            PageTable *l2 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
 
            if (!(pte2 & PTE_P))
                continue;
 
            for (k = 0; k < PAGETABLE_ENTRIES; k++) {
                PageEntry pte3 = l2->entries[k];
                PageTable *l3 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
 
                if (!(pte3 & PTE_P))
                    continue;
 
                if ((pte3 & PTE_PS) == 0) {
                    for (l = 0; l < PAGETABLE_ENTRIES; l++) {
                        PageEntry pte4 = l3->entries[l];
 
                        if (pte4 & PTE_P)
                            kprintf("0x%016llx: 0x%016llx P%c%c%c%c%c\n",
                                    AddrFromIJKL(i, j, k, l),
                                    (uint64_t)pte4,
                                    (pte4 & PTE_W) ? 'W' : ' ',
                                    (pte4 & PTE_NX) ? ' ' : 'X',
                                    (pte4 & PTE_U) ? 'U' : ' ',
                                    (pte4 & PTE_A) ? 'A' : ' ',
                                    (pte4 & PTE_D) ? 'D' : ' ');
                    }
                }
            }
        }
    }
 
    return;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_DumpFull()

void PMap_DumpFull

(

AS *

space

)

Definition at line 552 of file pmap.c.

{
    int i = 0;
    int j = 0;
    int k = 0;
    int l = 0;
    PageTable *root = space->root;
 
    kprintf("Root: %016llx\n", (uint64_t)space->root);
 
    for (i = 0; i < PAGETABLE_ENTRIES; i++) {
        PageEntry pte = root->entries[i];
        PageTable *l1 = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
        if (!(pte & PTE_P))
            continue;
 
        kprintf("Level 1: %016llx\n", (uint64_t)pte);
 
        for (j = 0; j < PAGETABLE_ENTRIES; j++) {
            PageEntry pte2 = l1->entries[j];
            PageTable *l2 = (PageTable *)DMPA2VA(pte2 & PGNUMMASK);
 
            if (!(pte2 & PTE_P))
                continue;
 
            kprintf("Level 2: %016llx\n", (uint64_t)pte2);
 
            for (k = 0; k < PAGETABLE_ENTRIES; k++) {
                PageEntry pte3 = l2->entries[k];
                PageTable *l3 = (PageTable *)DMPA2VA(pte3 & PGNUMMASK);
 
                if (!(pte3 & PTE_P))
                    continue;
 
                kprintf("Level 3: %016llx:%016llx\n",
                        AddrFromIJKL(i, j, k, 0),
                        (uint64_t)pte3);
 
                if ((pte3 & PTE_PS) == 0) {
                    for (l = 0; l < PAGETABLE_ENTRIES; l++) {
                        PageEntry pte4 = l3->entries[l];
 
                        kprintf("Level 4: %016llx:%016llx\n",
                                AddrFromIJKL(i, j, k, l),
                                (uint64_t)pte4);
                    }
                }
            }
        }
    }
 
    return;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_Init()

void PMap_Init

(

)

Definition at line 19 of file pmap.c.

{
    int i, j;
 
    kprintf("Initializing PMAP ... ");
 
    // Setup global state
    for (i = 0; i < MAX_CPUS; i++) {
        currentAS[i] = 0;
    }
 
    // Allocate system page table
    systemAS.root = PAlloc_AllocPage();
    systemAS.tables = PAGETABLE_ENTRIES / 2 + 1;
    systemAS.mappings = 0;
    if (!systemAS.root)
        PANIC("Cannot allocate system page table");
 
    for (i = 0; i < PAGETABLE_ENTRIES / 2; i++)
        systemAS.root->entries[i] = 0;
 
    for (i = PAGETABLE_ENTRIES / 2; i < PAGETABLE_ENTRIES; i++) {
        PageTable *pgtbl = PAlloc_AllocPage();
        PageEntry pte = DMVA2PA((uint64_t)pgtbl) | PTE_W | PTE_P;
        if (!pgtbl)
            PANIC("Not enough memory!");
 
        systemAS.root->entries[i] = pte;
 
        for (j = 0; j < PAGETABLE_ENTRIES; j++) {
            pgtbl->entries[j] = 0;
        }
    }
 
    // Setup system mappings
    PMap_SystemLMap(0x0, MEM_DIRECTMAP_BASE + 0x0,
                    3*512, 0); // 3GB RWX
    PMap_SystemLMap(0xC0000000, MEM_DIRECTMAP_BASE + 0xC0000000,
                    512, PTE_NX|PTE_PCD); // 1GB RW + PCD
    PMap_SystemLMap(0x100000000, MEM_DIRECTMAP_BASE + 0x100000000,
                    60*512, 0); // 60GB RWX
 
    PMap_LoadAS(&systemAS);
 
    kprintf("Done!\n");
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_InitAP()

void PMap_InitAP

(

)

Definition at line 67 of file pmap.c.

{
    PMap_LoadAS(&systemAS);
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_LoadAS()

void PMap_LoadAS

(

AS *

space

)

PMap_LoadAS –

Load an address space into the CPU. Reloads the CR3 register in x86-64 that points the physical page tables and flushes the TLB entries.

Parameters

[in]

space

Address space to load.

Definition at line 182 of file pmap.c.

{
    write_cr3(DMVA2PA((uint64_t)space->root));
    currentAS[THISCPU()] = space;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_Map()

bool PMap_Map	(	AS *	as,
		uint64_t	phys,
		uint64_t	virt,
		uint64_t	pages,
		uint64_t	flags
	)

PMap_Map –

Map a physical to virtual mapping in an address space.

Parameters

[in]	as	Address space.
[in]	phys	Physical address.
[in]	virt	Virtual address.
[in]	pages	Pages to map in.
[in]	flags	Flags to apply to the mapping.

Return values

true	On success
false	On failure

Definition at line 355 of file pmap.c.

{
    int i;
    PageEntry *entry;
 
    for (i = 0; i < pages; i++) {
        uint64_t va = virt + PGSIZE * i;
        PMapLookupEntry(as, va, &entry, PGSIZE);
        if (!entry) {
            kprintf("Map failed to allocate memory!\n");
            return false;
        }
 
        *entry = (phys + PGSIZE * i) | PTE_P | PTE_W | PTE_U | flags;
    }
 
    return true;
}

Here is the call graph for this function:

PMap_NewAS()

AS * PMap_NewAS

(

)

PMap_NewAS –

Create a new address space.

Returns

Newly created address space.

Definition at line 80 of file pmap.c.

{
    int i;
    AS *as = PAlloc_AllocPage();
 
    if (!as)
        return NULL;
 
    as->root = PAlloc_AllocPage();
    as->tables = 1;
    as->mappings = 0;
 
    if (!as->root) {
        PAlloc_Release(as);
        return NULL;
    }
 
    for (i = 0; i < PAGETABLE_ENTRIES / 2; i++)
    {
        as->root->entries[i] = 0;
    }
    for (i = PAGETABLE_ENTRIES / 2; i < PAGETABLE_ENTRIES; i++) {
        as->root->entries[i] = systemAS.root->entries[i];
    }
 
    return as;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_SystemLMap()

bool PMap_SystemLMap	(	uint64_t	phys,
		uint64_t	virt,
		uint64_t	lpages,
		uint64_t	flags
	)

PMap_SystemLMap –

Map a range of large (2MB) physical pages to virtual pages in the kernel address space that is shared by all processes.

Parameters

[in]	phys	Physical address.
[in]	virt	Virtual address.
[in]	lpages	Large pages to map in.
[in]	flags	Flags to apply to the mapping.

Return values

true	On success
false	On failure

Definition at line 480 of file pmap.c.

{
    int i;
    PageEntry *entry;
 
    for (i = 0; i < lpages; i++) {
        uint64_t va = virt + LARGE_PGSIZE * i;
        PMapLookupEntry(&systemAS, va, &entry, LARGE_PGSIZE);
        if (!entry) {
            kprintf("SystemLMap failed to allocate memory!\n");
            return false;
        }
 
        *entry = (phys + LARGE_PGSIZE * i) | PTE_P | PTE_W | PTE_PS | flags;
    }
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_SystemLookup()

void PMap_SystemLookup	(	uint64_t	va,
		PageEntry **	entry,
		int	size
	)

PMap_SystemLookup –

Lookup a kernel virtual address in a page table and return a pointer to the page entry. This function allocates page tables as necessary to fill in the 4-level heirarchy.

Parameters

[in]	va	Virtual address to lookup.
[out]	entry	Pointer will point to the PageEntry.
[in]	size	Page size we want to use.

Definition at line 460 of file pmap.c.

{
    PMapLookupEntry(&systemAS, va, entry, size);
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_SystemMap()

bool PMap_SystemMap	(	uint64_t	phys,
		uint64_t	virt,
		uint64_t	pages,
		uint64_t	flags
	)

PMap_SystemLMap –

Map a range of physical pages to virtual pages in the kernel address space that is shared by all processes.

Parameters

[in]	phys	Physical address.
[in]	virt	Virtual address.
[in]	pages	Pages to map in.
[in]	flags	Flags to apply to the mapping.

Return values

true	On success
false	On failure

Definition at line 514 of file pmap.c.

{
    int i;
    PageEntry *entry;
 
    for (i = 0; i < pages; i++) {
        uint64_t va = virt + PGSIZE * i;
        PMapLookupEntry(&systemAS, va, &entry, PGSIZE);
        if (!entry) {
            kprintf("SystemMap failed to allocate memory!\n");
            return false;
        }
 
        *entry = (phys + PGSIZE * i) | PTE_P | PTE_W | flags;
    }
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMap_SystemUnmap()

bool PMap_SystemUnmap	(	uint64_t	virt,
		uint64_t	pages
	)

PMap_SystemUnmap –

We do not currently use this!

Definition at line 539 of file pmap.c.

{
    NOT_IMPLEMENTED();
    return false;
}

PMap_Translate()

uintptr_t PMap_Translate	(	AS *	space,
		uintptr_t	va
	)

PMap_Translate –

Translates a virtual address to physical address for a given address space.

Parameters

[in]	space	Address space we wish to lookup a mapping in.
[in]	va	Virtual address we wish to translate.

Definition at line 220 of file pmap.c.

{
    int i,j,k,l;
    PageTable *table = space->root;
    PageEntry pte;
    PageEntry *entry;
 
    i = (va >> (HUGE_PGSHIFT + PGIDXSHIFT)) & PGIDXMASK;
    j = (va >> HUGE_PGSHIFT) & PGIDXMASK;
    k = (va >> LARGE_PGSHIFT) & PGIDXMASK;
    l = (va >> PGSHIFT) & PGIDXMASK;
 
    pte = table->entries[i];
    if (pte == 0) {
        ASSERT(pte);
        return 0;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    pte = table->entries[j];
    // XXX: Support 1GB pages
    if (pte == 0) {
        ASSERT(pte);
        return 0;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    pte = table->entries[k];
    if ((pte & PTE_PS) == PTE_PS) {
        // Handle 2MB pages
        entry = &table->entries[k];
        return (*entry & ~(LARGE_PGMASK | PTE_NX)) + (va & LARGE_PGMASK);
    }
    if (pte == 0) {
        ASSERT(pte);
        return 0;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    // Handle 4KB pages
    entry = &table->entries[l];
 
    return (*entry & ~(PGMASK | PTE_NX)) + (va & PGMASK);
}

Here is the caller graph for this function:

PMap_Unmap()

bool PMap_Unmap	(	AS *	as,
		uint64_t	va,
		uint64_t	pages
	)

PMap_Unmap –

Unmap a range of addresses.

Parameters

[in]	as	Address space.
[in]	va	Virtual address.
[in]	pages	Pages to map in.

Return values

true	On success
false	On failure

Definition at line 387 of file pmap.c.

{
    int i;
    PageEntry *entry;
 
    for (i = 0; i < pages; i++) {
        uint64_t vai = va + PGSIZE * i;
        PMapLookupEntry(as, vai, &entry, PGSIZE);
        if (!entry) {
            kprintf("Unmap tried to allocate memory!\n");
            return false;
        }
 
        NOT_IMPLEMENTED();
 
        *entry = 0;
    }
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMapAllocPageTable()

static PageTable * PMapAllocPageTable ( )

static

PMapAllocPageTable –

Allocates and initializes a page table.

Returns

Newly created PageTable.

Definition at line 196 of file pmap.c.

{
    int i;
    PageTable *pgtbl = PAlloc_AllocPage();
 
    if (!pgtbl)
        return 0;
 
    for (i = 0; i < PAGETABLE_ENTRIES; i++) {
        pgtbl->entries[i] = 0;
    }
 
    return pgtbl;
}

Here is the call graph for this function:

Here is the caller graph for this function:

PMapLookupEntry()

static void PMapLookupEntry ( AS *  space, uint64_t  va, PageEntry **  entry, int  size  )

static

PMapLookupEntry –

Lookup a virtual address in a page table and return a pointer to the page entry. This function allocates page tables as necessary to fill in the 4-level heirarchy.

Parameters

[in]	space	Address space to search.
[in]	va	Virtual address to lookup.
[out]	entry	Pointer will point to the PageEntry.
[in]	size	Page size we want to use.

Definition at line 278 of file pmap.c.

{
    int i,j,k,l;
    PageTable *table = space->root;
    PageEntry pte;
 
    i = (va >> (HUGE_PGSHIFT + PGIDXSHIFT)) & PGIDXMASK;
    j = (va >> HUGE_PGSHIFT) & PGIDXMASK;
    k = (va >> LARGE_PGSHIFT) & PGIDXMASK;
    l = (va >> PGSHIFT) & PGIDXMASK;
 
    *entry = NULL;
 
    pte = table->entries[i];
    if (pte == 0) {
        PageTable *newtable = PMapAllocPageTable();
        if (!newtable)
            return;
 
        pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
        table->entries[i] = pte;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    pte = table->entries[j];
    if (size == HUGE_PGSIZE) {
        // Handle 1GB pages
        *entry = &table->entries[j];
        return;
    }
    if (pte == 0) {
        PageTable *newtable = PMapAllocPageTable();
        if (!newtable)
            return;
 
        pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
        table->entries[j] = pte;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    pte = table->entries[k];
    if (size == LARGE_PGSIZE) {
        // Handle 2MB pages
        *entry = &table->entries[k];
        return;
    }
    if (pte == 0) {
        PageTable *newtable = PMapAllocPageTable();
        if (!newtable)
            return;
 
        pte = DMVA2PA((uint64_t)newtable) | PTE_P | PTE_W | PTE_U;
        table->entries[k] = pte;
    }
    table = (PageTable *)DMPA2VA(pte & PGNUMMASK);
 
    // Handle 4KB pages
    ASSERT(size == PGSIZE);
    *entry = &table->entries[l];
    return;
}

Here is the call graph for this function:

Here is the caller graph for this function:

REGISTER_DBGCMD() [1/2]

REGISTER_DBGCMD	(	pmapdump	,
		"Dump memory mappings"	,
		Debug_PMapDump
	)

REGISTER_DBGCMD() [2/2]

REGISTER_DBGCMD	(	pmapdumpfull	,
		"Dump memory mappings"	,
		Debug_PMapDumpFull
	)

Variable Documentation

currentAS

AS* currentAS[MAX_CPUS]

Definition at line 16 of file pmap.c.

systemAS

AS systemAS

Definition at line 15 of file pmap.c.