1231200Smm/* SPDX-License-Identifier: GPL-2.0 */ 2231200Smm#ifndef _ASM_X86_PGALLOC_H 3231200Smm#define _ASM_X86_PGALLOC_H 4231200Smm 5231200Smm#include <linux/threads.h> 6231200Smm#include <linux/mm.h> /* for struct page */ 7231200Smm#include <linux/pagemap.h> 8231200Smm 9231200Smm#define __HAVE_ARCH_PTE_ALLOC_ONE 10231200Smm#define __HAVE_ARCH_PGD_FREE 11231200Smm#include <asm-generic/pgalloc.h> 12231200Smm 13231200Smmstatic inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 14231200Smm 15231200Smm#ifdef CONFIG_PARAVIRT_XXL 16231200Smm#include <asm/paravirt.h> 17231200Smm#else 18231200Smm#define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 19231200Smmstatic inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 20231200Smmstatic inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 21231200Smmstatic inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 22231200Smmstatic inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 23231200Smm unsigned long start, unsigned long count) {} 24231200Smmstatic inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 25231200Smmstatic inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 26231200Smmstatic inline void paravirt_release_pte(unsigned long pfn) {} 27231200Smmstatic inline void paravirt_release_pmd(unsigned long pfn) {} 28231200Smmstatic inline void paravirt_release_pud(unsigned long pfn) {} 29302001Smmstatic inline void paravirt_release_p4d(unsigned long pfn) {} 30231200Smm#endif 31231200Smm 32231200Smm/* 33231200Smm * Flags to use when allocating a user page table page. 34231200Smm */ 35231200Smmextern gfp_t __userpte_alloc_gfp; 36231200Smm 37231200Smm#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION 38231200Smm/* 39231200Smm * Instead of one PGD, we acquire two PGDs. Being order-1, it is 40231200Smm * both 8k in size and 8k-aligned. That lets us just flip bit 12 41231200Smm * in a pointer to swap between the two 4k halves. 42231200Smm */ 43231200Smm#define PGD_ALLOCATION_ORDER 1 44231200Smm#else 45231200Smm#define PGD_ALLOCATION_ORDER 0 46231200Smm#endif 47231200Smm 48231200Smm/* 49231200Smm * Allocate and free page tables. 50231200Smm */ 51231200Smmextern pgd_t *pgd_alloc(struct mm_struct *); 52231200Smmextern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 53248616Smm 54231200Smmextern pgtable_t pte_alloc_one(struct mm_struct *); 55231200Smm 56231200Smmextern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 57231200Smm 58231200Smmstatic inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 59231200Smm unsigned long address) 60231200Smm{ 61231200Smm ___pte_free_tlb(tlb, pte); 62231200Smm} 63231200Smm 64231200Smmstatic inline void pmd_populate_kernel(struct mm_struct *mm, 65231200Smm pmd_t *pmd, pte_t *pte) 66231200Smm{ 67231200Smm paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 68231200Smm set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 69231200Smm} 70231200Smm 71231200Smmstatic inline void pmd_populate_kernel_safe(struct mm_struct *mm, 72231200Smm pmd_t *pmd, pte_t *pte) 73231200Smm{ 74231200Smm paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 75231200Smm set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 76231200Smm} 77231200Smm 78231200Smmstatic inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 79231200Smm struct page *pte) 80231200Smm{ 81231200Smm unsigned long pfn = page_to_pfn(pte); 82231200Smm 83231200Smm paravirt_alloc_pte(mm, pfn); 84231200Smm set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 85231200Smm} 86231200Smm 87231200Smm#if CONFIG_PGTABLE_LEVELS > 2 88231200Smmextern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 89231200Smm 90231200Smmstatic inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 91231200Smm unsigned long address) 92231200Smm{ 93231200Smm ___pmd_free_tlb(tlb, pmd); 94231200Smm} 95231200Smm 96231200Smm#ifdef CONFIG_X86_PAE 97231200Smmextern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 98231200Smm#else /* !CONFIG_X86_PAE */ 99231200Smmstatic inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 100231200Smm{ 101231200Smm paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 102231200Smm set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 103231200Smm} 104231200Smm 105231200Smmstatic inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 106231200Smm{ 107231200Smm paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 108231200Smm set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd))); 109231200Smm} 110231200Smm#endif /* CONFIG_X86_PAE */ 111231200Smm 112231200Smm#if CONFIG_PGTABLE_LEVELS > 3 113231200Smmstatic inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 114231200Smm{ 115231200Smm paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 116231200Smm set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 117231200Smm} 118231200Smm 119231200Smmstatic inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 120308152Smm{ 121308152Smm paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 122231200Smm set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 123231200Smm} 124231200Smm 125231200Smmextern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 126231200Smm 127231200Smmstatic inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 128231200Smm unsigned long address) 129231200Smm{ 130231200Smm ___pud_free_tlb(tlb, pud); 131231200Smm} 132231200Smm 133231200Smm#if CONFIG_PGTABLE_LEVELS > 4 134231200Smmstatic inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 135231200Smm{ 136231200Smm if (!pgtable_l5_enabled()) 137308152Smm return; 138308152Smm paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 139308152Smm set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 140308152Smm} 141308152Smm 142308152Smmstatic inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 143308152Smm{ 144308152Smm if (!pgtable_l5_enabled()) 145308152Smm return; 146308152Smm paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 147308152Smm set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 148308152Smm} 149308152Smm 150231200Smmstatic inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) 151231200Smm{ 152231200Smm gfp_t gfp = GFP_KERNEL_ACCOUNT; 153231200Smm 154231200Smm if (mm == &init_mm) 155231200Smm gfp &= ~__GFP_ACCOUNT; 156231200Smm return (p4d_t *)get_zeroed_page(gfp); 157231200Smm} 158231200Smm 159231200Smmstatic inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) 160231200Smm{ 161231200Smm if (!pgtable_l5_enabled()) 162231200Smm return; 163231200Smm 164231200Smm BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); 165231200Smm free_page((unsigned long)p4d); 166231200Smm} 167231200Smm 168231200Smmextern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 169231200Smm 170231200Smmstatic inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 171231200Smm unsigned long address) 172231200Smm{ 173231200Smm if (pgtable_l5_enabled()) 174231200Smm ___p4d_free_tlb(tlb, p4d); 175231200Smm} 176231200Smm 177231200Smm#endif /* CONFIG_PGTABLE_LEVELS > 4 */ 178231200Smm#endif /* CONFIG_PGTABLE_LEVELS > 3 */ 179231200Smm#endif /* CONFIG_PGTABLE_LEVELS > 2 */ 180231200Smm 181231200Smm#endif /* _ASM_X86_PGALLOC_H */ 182231200Smm