1190681Snwhitehorn/*- 2190681Snwhitehorn * Copyright (c) 2001 The NetBSD Foundation, Inc. 3190681Snwhitehorn * All rights reserved. 4190681Snwhitehorn * 5190681Snwhitehorn * This code is derived from software contributed to The NetBSD Foundation 6190681Snwhitehorn * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 7190681Snwhitehorn * 8190681Snwhitehorn * Redistribution and use in source and binary forms, with or without 9190681Snwhitehorn * modification, are permitted provided that the following conditions 10190681Snwhitehorn * are met: 11190681Snwhitehorn * 1. Redistributions of source code must retain the above copyright 12190681Snwhitehorn * notice, this list of conditions and the following disclaimer. 13190681Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 14190681Snwhitehorn * notice, this list of conditions and the following disclaimer in the 15190681Snwhitehorn * documentation and/or other materials provided with the distribution. 16190681Snwhitehorn * 17190681Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18190681Snwhitehorn * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19190681Snwhitehorn * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20190681Snwhitehorn * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21190681Snwhitehorn * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22190681Snwhitehorn * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23190681Snwhitehorn * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24190681Snwhitehorn * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25190681Snwhitehorn * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26190681Snwhitehorn * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27190681Snwhitehorn * POSSIBILITY OF SUCH DAMAGE. 28190681Snwhitehorn */ 29190681Snwhitehorn/*- 30190681Snwhitehorn * Copyright (C) 1995, 1996 Wolfgang Solfrank. 31190681Snwhitehorn * Copyright (C) 1995, 1996 TooLs GmbH. 32190681Snwhitehorn * All rights reserved. 33190681Snwhitehorn * 34190681Snwhitehorn * Redistribution and use in source and binary forms, with or without 35190681Snwhitehorn * modification, are permitted provided that the following conditions 36190681Snwhitehorn * are met: 37190681Snwhitehorn * 1. Redistributions of source code must retain the above copyright 38190681Snwhitehorn * notice, this list of conditions and the following disclaimer. 39190681Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 40190681Snwhitehorn * notice, this list of conditions and the following disclaimer in the 41190681Snwhitehorn * documentation and/or other materials provided with the distribution. 42190681Snwhitehorn * 3. All advertising materials mentioning features or use of this software 43190681Snwhitehorn * must display the following acknowledgement: 44190681Snwhitehorn * This product includes software developed by TooLs GmbH. 45190681Snwhitehorn * 4. The name of TooLs GmbH may not be used to endorse or promote products 46190681Snwhitehorn * derived from this software without specific prior written permission. 47190681Snwhitehorn * 48190681Snwhitehorn * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 49190681Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 50190681Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 51190681Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52190681Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 53190681Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 54190681Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 55190681Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 56190681Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 57190681Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58190681Snwhitehorn * 59190681Snwhitehorn * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 60190681Snwhitehorn */ 61190681Snwhitehorn/*- 62190681Snwhitehorn * Copyright (C) 2001 Benno Rice. 63190681Snwhitehorn * All rights reserved. 64190681Snwhitehorn * 65190681Snwhitehorn * Redistribution and use in source and binary forms, with or without 66190681Snwhitehorn * modification, are permitted provided that the following conditions 67190681Snwhitehorn * are met: 68190681Snwhitehorn * 1. Redistributions of source code must retain the above copyright 69190681Snwhitehorn * notice, this list of conditions and the following disclaimer. 70190681Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 71190681Snwhitehorn * notice, this list of conditions and the following disclaimer in the 72190681Snwhitehorn * documentation and/or other materials provided with the distribution. 73190681Snwhitehorn * 74190681Snwhitehorn * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 75190681Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 76190681Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 77190681Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 78190681Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 79190681Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 80190681Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 81190681Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 82190681Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 83190681Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 84190681Snwhitehorn */ 85190681Snwhitehorn 86190681Snwhitehorn#include <sys/cdefs.h> 87190681Snwhitehorn__FBSDID("$FreeBSD$"); 88190681Snwhitehorn 89190681Snwhitehorn/* 90190681Snwhitehorn * Manages physical address maps. 91190681Snwhitehorn * 92190681Snwhitehorn * Since the information managed by this module is also stored by the 93190681Snwhitehorn * logical address mapping module, this module may throw away valid virtual 94190681Snwhitehorn * to physical mappings at almost any time. However, invalidations of 95190681Snwhitehorn * mappings must be done as requested. 96190681Snwhitehorn * 97190681Snwhitehorn * In order to cope with hardware architectures which make virtual to 98190681Snwhitehorn * physical map invalidates expensive, this module may delay invalidate 99190681Snwhitehorn * reduced protection operations until such time as they are actually 100190681Snwhitehorn * necessary. This module is given full information as to which processors 101190681Snwhitehorn * are currently using which maps, and to when physical maps must be made 102190681Snwhitehorn * correct. 103190681Snwhitehorn */ 104190681Snwhitehorn 105230779Skib#include "opt_compat.h" 106190681Snwhitehorn#include "opt_kstack_pages.h" 107190681Snwhitehorn 108190681Snwhitehorn#include <sys/param.h> 109190681Snwhitehorn#include <sys/kernel.h> 110222813Sattilio#include <sys/queue.h> 111222813Sattilio#include <sys/cpuset.h> 112190681Snwhitehorn#include <sys/ktr.h> 113190681Snwhitehorn#include <sys/lock.h> 114190681Snwhitehorn#include <sys/msgbuf.h> 115243040Skib#include <sys/malloc.h> 116190681Snwhitehorn#include <sys/mutex.h> 117190681Snwhitehorn#include <sys/proc.h> 118233529Snwhitehorn#include <sys/rwlock.h> 119222813Sattilio#include <sys/sched.h> 120190681Snwhitehorn#include <sys/sysctl.h> 121190681Snwhitehorn#include <sys/systm.h> 122190681Snwhitehorn#include <sys/vmmeter.h> 123190681Snwhitehorn 124190681Snwhitehorn#include <sys/kdb.h> 125190681Snwhitehorn 126190681Snwhitehorn#include <dev/ofw/openfirm.h> 127190681Snwhitehorn 128190681Snwhitehorn#include <vm/vm.h> 129190681Snwhitehorn#include <vm/vm_param.h> 130190681Snwhitehorn#include <vm/vm_kern.h> 131190681Snwhitehorn#include <vm/vm_page.h> 132190681Snwhitehorn#include <vm/vm_map.h> 133190681Snwhitehorn#include <vm/vm_object.h> 134190681Snwhitehorn#include <vm/vm_extern.h> 135190681Snwhitehorn#include <vm/vm_pageout.h> 136190681Snwhitehorn#include <vm/uma.h> 137190681Snwhitehorn 138209975Snwhitehorn#include <machine/_inttypes.h> 139190681Snwhitehorn#include <machine/cpu.h> 140192067Snwhitehorn#include <machine/platform.h> 141190681Snwhitehorn#include <machine/frame.h> 142190681Snwhitehorn#include <machine/md_var.h> 143190681Snwhitehorn#include <machine/psl.h> 144190681Snwhitehorn#include <machine/bat.h> 145209975Snwhitehorn#include <machine/hid.h> 146190681Snwhitehorn#include <machine/pte.h> 147190681Snwhitehorn#include <machine/sr.h> 148190681Snwhitehorn#include <machine/trap.h> 149190681Snwhitehorn#include <machine/mmuvar.h> 150190681Snwhitehorn 151216174Snwhitehorn#include "mmu_oea64.h" 152190681Snwhitehorn#include "mmu_if.h" 153216174Snwhitehorn#include "moea64_if.h" 154190681Snwhitehorn 155209975Snwhitehornvoid moea64_release_vsid(uint64_t vsid); 156209975Snwhitehornuintptr_t moea64_get_unique_vsid(void); 157190681Snwhitehorn 158222614Snwhitehorn#define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) 159222614Snwhitehorn#define ENABLE_TRANS(msr) mtmsr(msr) 160190681Snwhitehorn 161190681Snwhitehorn#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) 162190681Snwhitehorn#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) 163204268Snwhitehorn#define VSID_HASH_MASK 0x0000007fffffffffULL 164190681Snwhitehorn 165233529Snwhitehorn/* 166233529Snwhitehorn * Locking semantics: 167233529Snwhitehorn * -- Read lock: if no modifications are being made to either the PVO lists 168233529Snwhitehorn * or page table or if any modifications being made result in internal 169233529Snwhitehorn * changes (e.g. wiring, protection) such that the existence of the PVOs 170233529Snwhitehorn * is unchanged and they remain associated with the same pmap (in which 171233529Snwhitehorn * case the changes should be protected by the pmap lock) 172233529Snwhitehorn * -- Write lock: required if PTEs/PVOs are being inserted or removed. 173233529Snwhitehorn */ 174190681Snwhitehorn 175233529Snwhitehorn#define LOCK_TABLE_RD() rw_rlock(&moea64_table_lock) 176233529Snwhitehorn#define UNLOCK_TABLE_RD() rw_runlock(&moea64_table_lock) 177233529Snwhitehorn#define LOCK_TABLE_WR() rw_wlock(&moea64_table_lock) 178233529Snwhitehorn#define UNLOCK_TABLE_WR() rw_wunlock(&moea64_table_lock) 179233529Snwhitehorn 180190681Snwhitehornstruct ofw_map { 181209975Snwhitehorn cell_t om_va; 182209975Snwhitehorn cell_t om_len; 183266019Sian uint64_t om_pa; 184209975Snwhitehorn cell_t om_mode; 185190681Snwhitehorn}; 186190681Snwhitehorn 187260672Sjhibbitsextern unsigned char _etext[]; 188260672Sjhibbitsextern unsigned char _end[]; 189260672Sjhibbits 190260672Sjhibbitsextern int dumpsys_minidump; 191260672Sjhibbits 192190681Snwhitehorn/* 193190681Snwhitehorn * Map of physical memory regions. 194190681Snwhitehorn */ 195190681Snwhitehornstatic struct mem_region *regions; 196190681Snwhitehornstatic struct mem_region *pregions; 197209975Snwhitehornstatic u_int phys_avail_count; 198209975Snwhitehornstatic int regions_sz, pregions_sz; 199190681Snwhitehorn 200190681Snwhitehornextern void bs_remap_earlyboot(void); 201190681Snwhitehorn 202190681Snwhitehorn/* 203190681Snwhitehorn * Lock for the pteg and pvo tables. 204190681Snwhitehorn */ 205233529Snwhitehornstruct rwlock moea64_table_lock; 206211967Snwhitehornstruct mtx moea64_slb_mutex; 207190681Snwhitehorn 208190681Snwhitehorn/* 209190681Snwhitehorn * PTEG data. 210190681Snwhitehorn */ 211190681Snwhitehornu_int moea64_pteg_count; 212190681Snwhitehornu_int moea64_pteg_mask; 213190681Snwhitehorn 214190681Snwhitehorn/* 215190681Snwhitehorn * PVO data. 216190681Snwhitehorn */ 217190681Snwhitehornstruct pvo_head *moea64_pvo_table; /* pvo entries by pteg index */ 218190681Snwhitehorn 219190681Snwhitehornuma_zone_t moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */ 220190681Snwhitehornuma_zone_t moea64_mpvo_zone; /* zone for pvo entries for managed pages */ 221190681Snwhitehorn 222190681Snwhitehorn#define BPVO_POOL_SIZE 327680 223190681Snwhitehornstatic struct pvo_entry *moea64_bpvo_pool; 224190681Snwhitehornstatic int moea64_bpvo_pool_index = 0; 225190681Snwhitehorn 226190681Snwhitehorn#define VSID_NBPW (sizeof(u_int32_t) * 8) 227209975Snwhitehorn#ifdef __powerpc64__ 228209975Snwhitehorn#define NVSIDS (NPMAPS * 16) 229209975Snwhitehorn#define VSID_HASHMASK 0xffffffffUL 230209975Snwhitehorn#else 231209975Snwhitehorn#define NVSIDS NPMAPS 232209975Snwhitehorn#define VSID_HASHMASK 0xfffffUL 233209975Snwhitehorn#endif 234209975Snwhitehornstatic u_int moea64_vsid_bitmap[NVSIDS / VSID_NBPW]; 235190681Snwhitehorn 236190681Snwhitehornstatic boolean_t moea64_initialized = FALSE; 237190681Snwhitehorn 238190681Snwhitehorn/* 239190681Snwhitehorn * Statistics. 240190681Snwhitehorn */ 241190681Snwhitehornu_int moea64_pte_valid = 0; 242190681Snwhitehornu_int moea64_pte_overflow = 0; 243190681Snwhitehornu_int moea64_pvo_entries = 0; 244190681Snwhitehornu_int moea64_pvo_enter_calls = 0; 245190681Snwhitehornu_int moea64_pvo_remove_calls = 0; 246190681SnwhitehornSYSCTL_INT(_machdep, OID_AUTO, moea64_pte_valid, CTLFLAG_RD, 247190681Snwhitehorn &moea64_pte_valid, 0, ""); 248190681SnwhitehornSYSCTL_INT(_machdep, OID_AUTO, moea64_pte_overflow, CTLFLAG_RD, 249190681Snwhitehorn &moea64_pte_overflow, 0, ""); 250190681SnwhitehornSYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_entries, CTLFLAG_RD, 251190681Snwhitehorn &moea64_pvo_entries, 0, ""); 252190681SnwhitehornSYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_enter_calls, CTLFLAG_RD, 253190681Snwhitehorn &moea64_pvo_enter_calls, 0, ""); 254190681SnwhitehornSYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD, 255190681Snwhitehorn &moea64_pvo_remove_calls, 0, ""); 256190681Snwhitehorn 257190681Snwhitehornvm_offset_t moea64_scratchpage_va[2]; 258216174Snwhitehornstruct pvo_entry *moea64_scratchpage_pvo[2]; 259216174Snwhitehornuintptr_t moea64_scratchpage_pte[2]; 260190681Snwhitehornstruct mtx moea64_scratchpage_mtx; 261190681Snwhitehorn 262209975Snwhitehornuint64_t moea64_large_page_mask = 0; 263255418Snwhitehornuint64_t moea64_large_page_size = 0; 264209975Snwhitehornint moea64_large_page_shift = 0; 265209975Snwhitehorn 266190681Snwhitehorn/* 267190681Snwhitehorn * PVO calls. 268190681Snwhitehorn */ 269216174Snwhitehornstatic int moea64_pvo_enter(mmu_t, pmap_t, uma_zone_t, struct pvo_head *, 270270439Skib vm_offset_t, vm_offset_t, uint64_t, int, int8_t); 271216174Snwhitehornstatic void moea64_pvo_remove(mmu_t, struct pvo_entry *); 272209975Snwhitehornstatic struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t); 273190681Snwhitehorn 274190681Snwhitehorn/* 275190681Snwhitehorn * Utility routines. 276190681Snwhitehorn */ 277216174Snwhitehornstatic boolean_t moea64_query_bit(mmu_t, vm_page_t, u_int64_t); 278216174Snwhitehornstatic u_int moea64_clear_bit(mmu_t, vm_page_t, u_int64_t); 279190681Snwhitehornstatic void moea64_kremove(mmu_t, vm_offset_t); 280216174Snwhitehornstatic void moea64_syncicache(mmu_t, pmap_t pmap, vm_offset_t va, 281198341Smarcel vm_offset_t pa, vm_size_t sz); 282190681Snwhitehorn 283190681Snwhitehorn/* 284190681Snwhitehorn * Kernel MMU interface 285190681Snwhitehorn */ 286190681Snwhitehornvoid moea64_clear_modify(mmu_t, vm_page_t); 287190681Snwhitehornvoid moea64_copy_page(mmu_t, vm_page_t, vm_page_t); 288248280Skibvoid moea64_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 289248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize); 290270439Skibint moea64_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, 291270439Skib u_int flags, int8_t psind); 292190681Snwhitehornvoid moea64_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 293190681Snwhitehorn vm_prot_t); 294190681Snwhitehornvoid moea64_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 295190681Snwhitehornvm_paddr_t moea64_extract(mmu_t, pmap_t, vm_offset_t); 296190681Snwhitehornvm_page_t moea64_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t); 297190681Snwhitehornvoid moea64_init(mmu_t); 298190681Snwhitehornboolean_t moea64_is_modified(mmu_t, vm_page_t); 299214617Salcboolean_t moea64_is_prefaultable(mmu_t, pmap_t, vm_offset_t); 300207155Salcboolean_t moea64_is_referenced(mmu_t, vm_page_t); 301238357Salcint moea64_ts_referenced(mmu_t, vm_page_t); 302236019Srajvm_offset_t moea64_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 303190681Snwhitehornboolean_t moea64_page_exists_quick(mmu_t, pmap_t, vm_page_t); 304190681Snwhitehornint moea64_page_wired_mappings(mmu_t, vm_page_t); 305190681Snwhitehornvoid moea64_pinit(mmu_t, pmap_t); 306190681Snwhitehornvoid moea64_pinit0(mmu_t, pmap_t); 307190681Snwhitehornvoid moea64_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 308190681Snwhitehornvoid moea64_qenter(mmu_t, vm_offset_t, vm_page_t *, int); 309190681Snwhitehornvoid moea64_qremove(mmu_t, vm_offset_t, int); 310190681Snwhitehornvoid moea64_release(mmu_t, pmap_t); 311190681Snwhitehornvoid moea64_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 312233017Snwhitehornvoid moea64_remove_pages(mmu_t, pmap_t); 313190681Snwhitehornvoid moea64_remove_all(mmu_t, vm_page_t); 314190681Snwhitehornvoid moea64_remove_write(mmu_t, vm_page_t); 315270920Skibvoid moea64_unwire(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 316190681Snwhitehornvoid moea64_zero_page(mmu_t, vm_page_t); 317190681Snwhitehornvoid moea64_zero_page_area(mmu_t, vm_page_t, int, int); 318190681Snwhitehornvoid moea64_zero_page_idle(mmu_t, vm_page_t); 319190681Snwhitehornvoid moea64_activate(mmu_t, struct thread *); 320190681Snwhitehornvoid moea64_deactivate(mmu_t, struct thread *); 321236019Srajvoid *moea64_mapdev(mmu_t, vm_paddr_t, vm_size_t); 322213307Snwhitehornvoid *moea64_mapdev_attr(mmu_t, vm_offset_t, vm_size_t, vm_memattr_t); 323190681Snwhitehornvoid moea64_unmapdev(mmu_t, vm_offset_t, vm_size_t); 324236019Srajvm_paddr_t moea64_kextract(mmu_t, vm_offset_t); 325213307Snwhitehornvoid moea64_page_set_memattr(mmu_t, vm_page_t m, vm_memattr_t ma); 326213307Snwhitehornvoid moea64_kenter_attr(mmu_t, vm_offset_t, vm_offset_t, vm_memattr_t ma); 327236019Srajvoid moea64_kenter(mmu_t, vm_offset_t, vm_paddr_t); 328236019Srajboolean_t moea64_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t); 329198341Smarcelstatic void moea64_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t); 330260672Sjhibbitsvm_offset_t moea64_dumpsys_map(mmu_t mmu, struct pmap_md *md, vm_size_t ofs, 331260672Sjhibbits vm_size_t *sz); 332260672Sjhibbitsstruct pmap_md * moea64_scan_md(mmu_t mmu, struct pmap_md *prev); 333190681Snwhitehorn 334209975Snwhitehornstatic mmu_method_t moea64_methods[] = { 335190681Snwhitehorn MMUMETHOD(mmu_clear_modify, moea64_clear_modify), 336190681Snwhitehorn MMUMETHOD(mmu_copy_page, moea64_copy_page), 337248280Skib MMUMETHOD(mmu_copy_pages, moea64_copy_pages), 338190681Snwhitehorn MMUMETHOD(mmu_enter, moea64_enter), 339190681Snwhitehorn MMUMETHOD(mmu_enter_object, moea64_enter_object), 340190681Snwhitehorn MMUMETHOD(mmu_enter_quick, moea64_enter_quick), 341190681Snwhitehorn MMUMETHOD(mmu_extract, moea64_extract), 342190681Snwhitehorn MMUMETHOD(mmu_extract_and_hold, moea64_extract_and_hold), 343190681Snwhitehorn MMUMETHOD(mmu_init, moea64_init), 344190681Snwhitehorn MMUMETHOD(mmu_is_modified, moea64_is_modified), 345214617Salc MMUMETHOD(mmu_is_prefaultable, moea64_is_prefaultable), 346207155Salc MMUMETHOD(mmu_is_referenced, moea64_is_referenced), 347190681Snwhitehorn MMUMETHOD(mmu_ts_referenced, moea64_ts_referenced), 348190681Snwhitehorn MMUMETHOD(mmu_map, moea64_map), 349190681Snwhitehorn MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick), 350190681Snwhitehorn MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings), 351190681Snwhitehorn MMUMETHOD(mmu_pinit, moea64_pinit), 352190681Snwhitehorn MMUMETHOD(mmu_pinit0, moea64_pinit0), 353190681Snwhitehorn MMUMETHOD(mmu_protect, moea64_protect), 354190681Snwhitehorn MMUMETHOD(mmu_qenter, moea64_qenter), 355190681Snwhitehorn MMUMETHOD(mmu_qremove, moea64_qremove), 356190681Snwhitehorn MMUMETHOD(mmu_release, moea64_release), 357190681Snwhitehorn MMUMETHOD(mmu_remove, moea64_remove), 358233017Snwhitehorn MMUMETHOD(mmu_remove_pages, moea64_remove_pages), 359190681Snwhitehorn MMUMETHOD(mmu_remove_all, moea64_remove_all), 360190681Snwhitehorn MMUMETHOD(mmu_remove_write, moea64_remove_write), 361198341Smarcel MMUMETHOD(mmu_sync_icache, moea64_sync_icache), 362270920Skib MMUMETHOD(mmu_unwire, moea64_unwire), 363190681Snwhitehorn MMUMETHOD(mmu_zero_page, moea64_zero_page), 364190681Snwhitehorn MMUMETHOD(mmu_zero_page_area, moea64_zero_page_area), 365190681Snwhitehorn MMUMETHOD(mmu_zero_page_idle, moea64_zero_page_idle), 366190681Snwhitehorn MMUMETHOD(mmu_activate, moea64_activate), 367190681Snwhitehorn MMUMETHOD(mmu_deactivate, moea64_deactivate), 368213307Snwhitehorn MMUMETHOD(mmu_page_set_memattr, moea64_page_set_memattr), 369190681Snwhitehorn 370190681Snwhitehorn /* Internal interfaces */ 371190681Snwhitehorn MMUMETHOD(mmu_mapdev, moea64_mapdev), 372213307Snwhitehorn MMUMETHOD(mmu_mapdev_attr, moea64_mapdev_attr), 373190681Snwhitehorn MMUMETHOD(mmu_unmapdev, moea64_unmapdev), 374190681Snwhitehorn MMUMETHOD(mmu_kextract, moea64_kextract), 375190681Snwhitehorn MMUMETHOD(mmu_kenter, moea64_kenter), 376213307Snwhitehorn MMUMETHOD(mmu_kenter_attr, moea64_kenter_attr), 377190681Snwhitehorn MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped), 378260672Sjhibbits MMUMETHOD(mmu_scan_md, moea64_scan_md), 379260672Sjhibbits MMUMETHOD(mmu_dumpsys_map, moea64_dumpsys_map), 380190681Snwhitehorn 381190681Snwhitehorn { 0, 0 } 382190681Snwhitehorn}; 383190681Snwhitehorn 384216174SnwhitehornMMU_DEF(oea64_mmu, "mmu_oea64_base", moea64_methods, 0); 385190681Snwhitehorn 386190681Snwhitehornstatic __inline u_int 387209975Snwhitehornva_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 388190681Snwhitehorn{ 389204268Snwhitehorn uint64_t hash; 390209975Snwhitehorn int shift; 391190681Snwhitehorn 392209975Snwhitehorn shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 393204268Snwhitehorn hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 394209975Snwhitehorn shift); 395190681Snwhitehorn return (hash & moea64_pteg_mask); 396190681Snwhitehorn} 397190681Snwhitehorn 398190681Snwhitehornstatic __inline struct pvo_head * 399190681Snwhitehornvm_page_to_pvoh(vm_page_t m) 400190681Snwhitehorn{ 401190681Snwhitehorn 402190681Snwhitehorn return (&m->md.mdpg_pvoh); 403190681Snwhitehorn} 404190681Snwhitehorn 405190681Snwhitehornstatic __inline void 406190681Snwhitehornmoea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va, 407209975Snwhitehorn uint64_t pte_lo, int flags) 408190681Snwhitehorn{ 409209975Snwhitehorn 410190681Snwhitehorn /* 411190681Snwhitehorn * Construct a PTE. Default to IMB initially. Valid bit only gets 412190681Snwhitehorn * set when the real pte is set in memory. 413190681Snwhitehorn * 414190681Snwhitehorn * Note: Don't set the valid bit for correct operation of tlb update. 415190681Snwhitehorn */ 416190681Snwhitehorn pt->pte_hi = (vsid << LPTE_VSID_SHIFT) | 417190681Snwhitehorn (((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API); 418190681Snwhitehorn 419209975Snwhitehorn if (flags & PVO_LARGE) 420209975Snwhitehorn pt->pte_hi |= LPTE_BIG; 421209975Snwhitehorn 422190681Snwhitehorn pt->pte_lo = pte_lo; 423190681Snwhitehorn} 424190681Snwhitehorn 425190681Snwhitehornstatic __inline uint64_t 426213307Snwhitehornmoea64_calc_wimg(vm_offset_t pa, vm_memattr_t ma) 427190681Snwhitehorn{ 428190681Snwhitehorn uint64_t pte_lo; 429190681Snwhitehorn int i; 430190681Snwhitehorn 431213307Snwhitehorn if (ma != VM_MEMATTR_DEFAULT) { 432213307Snwhitehorn switch (ma) { 433213307Snwhitehorn case VM_MEMATTR_UNCACHEABLE: 434213307Snwhitehorn return (LPTE_I | LPTE_G); 435213307Snwhitehorn case VM_MEMATTR_WRITE_COMBINING: 436213307Snwhitehorn case VM_MEMATTR_WRITE_BACK: 437213307Snwhitehorn case VM_MEMATTR_PREFETCHABLE: 438213307Snwhitehorn return (LPTE_I); 439213307Snwhitehorn case VM_MEMATTR_WRITE_THROUGH: 440213307Snwhitehorn return (LPTE_W | LPTE_M); 441213307Snwhitehorn } 442213307Snwhitehorn } 443213307Snwhitehorn 444190681Snwhitehorn /* 445190681Snwhitehorn * Assume the page is cache inhibited and access is guarded unless 446190681Snwhitehorn * it's in our available memory array. 447190681Snwhitehorn */ 448190681Snwhitehorn pte_lo = LPTE_I | LPTE_G; 449190681Snwhitehorn for (i = 0; i < pregions_sz; i++) { 450190681Snwhitehorn if ((pa >= pregions[i].mr_start) && 451190681Snwhitehorn (pa < (pregions[i].mr_start + pregions[i].mr_size))) { 452190681Snwhitehorn pte_lo &= ~(LPTE_I | LPTE_G); 453190681Snwhitehorn pte_lo |= LPTE_M; 454190681Snwhitehorn break; 455190681Snwhitehorn } 456190681Snwhitehorn } 457190681Snwhitehorn 458190681Snwhitehorn return pte_lo; 459190681Snwhitehorn} 460190681Snwhitehorn 461190681Snwhitehorn/* 462190681Snwhitehorn * Quick sort callout for comparing memory regions. 463190681Snwhitehorn */ 464190681Snwhitehornstatic int om_cmp(const void *a, const void *b); 465190681Snwhitehorn 466190681Snwhitehornstatic int 467190681Snwhitehornom_cmp(const void *a, const void *b) 468190681Snwhitehorn{ 469190681Snwhitehorn const struct ofw_map *mapa; 470190681Snwhitehorn const struct ofw_map *mapb; 471190681Snwhitehorn 472190681Snwhitehorn mapa = a; 473190681Snwhitehorn mapb = b; 474266019Sian if (mapa->om_pa < mapb->om_pa) 475190681Snwhitehorn return (-1); 476266019Sian else if (mapa->om_pa > mapb->om_pa) 477190681Snwhitehorn return (1); 478190681Snwhitehorn else 479190681Snwhitehorn return (0); 480190681Snwhitehorn} 481190681Snwhitehorn 482190681Snwhitehornstatic void 483199226Snwhitehornmoea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz) 484199226Snwhitehorn{ 485266019Sian struct ofw_map translations[sz/(4*sizeof(cell_t))]; /*>= 4 cells per */ 486266019Sian pcell_t acells, trans_cells[sz/sizeof(cell_t)]; 487199226Snwhitehorn register_t msr; 488199226Snwhitehorn vm_offset_t off; 489204128Snwhitehorn vm_paddr_t pa_base; 490266019Sian int i, j; 491199226Snwhitehorn 492199226Snwhitehorn bzero(translations, sz); 493266019Sian OF_getprop(OF_finddevice("/"), "#address-cells", &acells, 494266019Sian sizeof(acells)); 495266019Sian if (OF_getprop(mmu, "translations", trans_cells, sz) == -1) 496199226Snwhitehorn panic("moea64_bootstrap: can't get ofw translations"); 497199226Snwhitehorn 498199226Snwhitehorn CTR0(KTR_PMAP, "moea64_add_ofw_mappings: translations"); 499266019Sian sz /= sizeof(cell_t); 500266019Sian for (i = 0, j = 0; i < sz; j++) { 501266019Sian translations[j].om_va = trans_cells[i++]; 502266019Sian translations[j].om_len = trans_cells[i++]; 503266019Sian translations[j].om_pa = trans_cells[i++]; 504266019Sian if (acells == 2) { 505266019Sian translations[j].om_pa <<= 32; 506266019Sian translations[j].om_pa |= trans_cells[i++]; 507266019Sian } 508266019Sian translations[j].om_mode = trans_cells[i++]; 509266019Sian } 510266019Sian KASSERT(i == sz, ("Translations map has incorrect cell count (%d/%zd)", 511266019Sian i, sz)); 512266019Sian 513266019Sian sz = j; 514199226Snwhitehorn qsort(translations, sz, sizeof (*translations), om_cmp); 515199226Snwhitehorn 516216563Snwhitehorn for (i = 0; i < sz; i++) { 517266019Sian pa_base = translations[i].om_pa; 518266019Sian #ifndef __powerpc64__ 519266019Sian if ((translations[i].om_pa >> 32) != 0) 520199226Snwhitehorn panic("OFW translations above 32-bit boundary!"); 521209975Snwhitehorn #endif 522199226Snwhitehorn 523265998Sian if (pa_base % PAGE_SIZE) 524265998Sian panic("OFW translation not page-aligned (phys)!"); 525265998Sian if (translations[i].om_va % PAGE_SIZE) 526265998Sian panic("OFW translation not page-aligned (virt)!"); 527265998Sian 528265998Sian CTR3(KTR_PMAP, "translation: pa=%#zx va=%#x len=%#x", 529265998Sian pa_base, translations[i].om_va, translations[i].om_len); 530265998Sian 531199226Snwhitehorn /* Now enter the pages for this mapping */ 532199226Snwhitehorn 533199226Snwhitehorn DISABLE_TRANS(msr); 534199226Snwhitehorn for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) { 535209975Snwhitehorn if (moea64_pvo_find_va(kernel_pmap, 536209975Snwhitehorn translations[i].om_va + off) != NULL) 537209975Snwhitehorn continue; 538209975Snwhitehorn 539204128Snwhitehorn moea64_kenter(mmup, translations[i].om_va + off, 540204128Snwhitehorn pa_base + off); 541199226Snwhitehorn } 542199226Snwhitehorn ENABLE_TRANS(msr); 543199226Snwhitehorn } 544199226Snwhitehorn} 545199226Snwhitehorn 546209975Snwhitehorn#ifdef __powerpc64__ 547199226Snwhitehornstatic void 548209975Snwhitehornmoea64_probe_large_page(void) 549190681Snwhitehorn{ 550209975Snwhitehorn uint16_t pvr = mfpvr() >> 16; 551209975Snwhitehorn 552209975Snwhitehorn switch (pvr) { 553209975Snwhitehorn case IBM970: 554209975Snwhitehorn case IBM970FX: 555209975Snwhitehorn case IBM970MP: 556209975Snwhitehorn powerpc_sync(); isync(); 557209975Snwhitehorn mtspr(SPR_HID4, mfspr(SPR_HID4) & ~HID4_970_DISABLE_LG_PG); 558209975Snwhitehorn powerpc_sync(); isync(); 559209975Snwhitehorn 560209975Snwhitehorn /* FALLTHROUGH */ 561255418Snwhitehorn default: 562209975Snwhitehorn moea64_large_page_size = 0x1000000; /* 16 MB */ 563209975Snwhitehorn moea64_large_page_shift = 24; 564209975Snwhitehorn } 565209975Snwhitehorn 566209975Snwhitehorn moea64_large_page_mask = moea64_large_page_size - 1; 567209975Snwhitehorn} 568209975Snwhitehorn 569209975Snwhitehornstatic void 570209975Snwhitehornmoea64_bootstrap_slb_prefault(vm_offset_t va, int large) 571209975Snwhitehorn{ 572209975Snwhitehorn struct slb *cache; 573209975Snwhitehorn struct slb entry; 574209975Snwhitehorn uint64_t esid, slbe; 575209975Snwhitehorn uint64_t i; 576209975Snwhitehorn 577209975Snwhitehorn cache = PCPU_GET(slb); 578209975Snwhitehorn esid = va >> ADDR_SR_SHFT; 579209975Snwhitehorn slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; 580209975Snwhitehorn 581209975Snwhitehorn for (i = 0; i < 64; i++) { 582209975Snwhitehorn if (cache[i].slbe == (slbe | i)) 583209975Snwhitehorn return; 584209975Snwhitehorn } 585209975Snwhitehorn 586209975Snwhitehorn entry.slbe = slbe; 587210704Snwhitehorn entry.slbv = KERNEL_VSID(esid) << SLBV_VSID_SHIFT; 588209975Snwhitehorn if (large) 589209975Snwhitehorn entry.slbv |= SLBV_L; 590209975Snwhitehorn 591212722Snwhitehorn slb_insert_kernel(entry.slbe, entry.slbv); 592209975Snwhitehorn} 593209975Snwhitehorn#endif 594209975Snwhitehorn 595209975Snwhitehornstatic void 596209975Snwhitehornmoea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart, 597209975Snwhitehorn vm_offset_t kernelend) 598209975Snwhitehorn{ 599209975Snwhitehorn register_t msr; 600209975Snwhitehorn vm_paddr_t pa; 601209975Snwhitehorn vm_offset_t size, off; 602209975Snwhitehorn uint64_t pte_lo; 603209975Snwhitehorn int i; 604209975Snwhitehorn 605209975Snwhitehorn if (moea64_large_page_size == 0) 606209975Snwhitehorn hw_direct_map = 0; 607209975Snwhitehorn 608209975Snwhitehorn DISABLE_TRANS(msr); 609209975Snwhitehorn if (hw_direct_map) { 610233529Snwhitehorn LOCK_TABLE_WR(); 611209975Snwhitehorn PMAP_LOCK(kernel_pmap); 612209975Snwhitehorn for (i = 0; i < pregions_sz; i++) { 613209975Snwhitehorn for (pa = pregions[i].mr_start; pa < pregions[i].mr_start + 614209975Snwhitehorn pregions[i].mr_size; pa += moea64_large_page_size) { 615209975Snwhitehorn pte_lo = LPTE_M; 616209975Snwhitehorn 617209975Snwhitehorn /* 618209975Snwhitehorn * Set memory access as guarded if prefetch within 619209975Snwhitehorn * the page could exit the available physmem area. 620209975Snwhitehorn */ 621209975Snwhitehorn if (pa & moea64_large_page_mask) { 622209975Snwhitehorn pa &= moea64_large_page_mask; 623209975Snwhitehorn pte_lo |= LPTE_G; 624209975Snwhitehorn } 625209975Snwhitehorn if (pa + moea64_large_page_size > 626209975Snwhitehorn pregions[i].mr_start + pregions[i].mr_size) 627209975Snwhitehorn pte_lo |= LPTE_G; 628209975Snwhitehorn 629216174Snwhitehorn moea64_pvo_enter(mmup, kernel_pmap, moea64_upvo_zone, 630235689Snwhitehorn NULL, pa, pa, pte_lo, 631270439Skib PVO_WIRED | PVO_LARGE, 0); 632209975Snwhitehorn } 633209975Snwhitehorn } 634209975Snwhitehorn PMAP_UNLOCK(kernel_pmap); 635233529Snwhitehorn UNLOCK_TABLE_WR(); 636209975Snwhitehorn } else { 637209975Snwhitehorn size = sizeof(struct pvo_head) * moea64_pteg_count; 638209975Snwhitehorn off = (vm_offset_t)(moea64_pvo_table); 639209975Snwhitehorn for (pa = off; pa < off + size; pa += PAGE_SIZE) 640209975Snwhitehorn moea64_kenter(mmup, pa, pa); 641209975Snwhitehorn size = BPVO_POOL_SIZE*sizeof(struct pvo_entry); 642209975Snwhitehorn off = (vm_offset_t)(moea64_bpvo_pool); 643209975Snwhitehorn for (pa = off; pa < off + size; pa += PAGE_SIZE) 644209975Snwhitehorn moea64_kenter(mmup, pa, pa); 645209975Snwhitehorn 646209975Snwhitehorn /* 647209975Snwhitehorn * Map certain important things, like ourselves. 648209975Snwhitehorn * 649209975Snwhitehorn * NOTE: We do not map the exception vector space. That code is 650209975Snwhitehorn * used only in real mode, and leaving it unmapped allows us to 651209975Snwhitehorn * catch NULL pointer deferences, instead of making NULL a valid 652209975Snwhitehorn * address. 653209975Snwhitehorn */ 654209975Snwhitehorn 655209975Snwhitehorn for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; 656209975Snwhitehorn pa += PAGE_SIZE) 657209975Snwhitehorn moea64_kenter(mmup, pa, pa); 658209975Snwhitehorn } 659209975Snwhitehorn ENABLE_TRANS(msr); 660248508Skib 661248508Skib /* 662248508Skib * Allow user to override unmapped_buf_allowed for testing. 663248508Skib * XXXKIB Only direct map implementation was tested. 664248508Skib */ 665248508Skib if (!TUNABLE_INT_FETCH("vfs.unmapped_buf_allowed", 666248508Skib &unmapped_buf_allowed)) 667248508Skib unmapped_buf_allowed = hw_direct_map; 668209975Snwhitehorn} 669209975Snwhitehorn 670216174Snwhitehornvoid 671216174Snwhitehornmoea64_early_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 672209975Snwhitehorn{ 673190681Snwhitehorn int i, j; 674216174Snwhitehorn vm_size_t physsz, hwphyssz; 675190681Snwhitehorn 676209975Snwhitehorn#ifndef __powerpc64__ 677190681Snwhitehorn /* We don't have a direct map since there is no BAT */ 678190681Snwhitehorn hw_direct_map = 0; 679190681Snwhitehorn 680190681Snwhitehorn /* Make sure battable is zero, since we have no BAT */ 681190681Snwhitehorn for (i = 0; i < 16; i++) { 682190681Snwhitehorn battable[i].batu = 0; 683190681Snwhitehorn battable[i].batl = 0; 684190681Snwhitehorn } 685209975Snwhitehorn#else 686209975Snwhitehorn moea64_probe_large_page(); 687190681Snwhitehorn 688209975Snwhitehorn /* Use a direct map if we have large page support */ 689209975Snwhitehorn if (moea64_large_page_size > 0) 690209975Snwhitehorn hw_direct_map = 1; 691209975Snwhitehorn else 692209975Snwhitehorn hw_direct_map = 0; 693209975Snwhitehorn#endif 694209975Snwhitehorn 695190681Snwhitehorn /* Get physical memory regions from firmware */ 696190681Snwhitehorn mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 697190681Snwhitehorn CTR0(KTR_PMAP, "moea64_bootstrap: physical memory"); 698190681Snwhitehorn 699190681Snwhitehorn if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) 700190681Snwhitehorn panic("moea64_bootstrap: phys_avail too small"); 701222614Snwhitehorn 702190681Snwhitehorn phys_avail_count = 0; 703190681Snwhitehorn physsz = 0; 704190681Snwhitehorn hwphyssz = 0; 705190681Snwhitehorn TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 706190681Snwhitehorn for (i = 0, j = 0; i < regions_sz; i++, j += 2) { 707265998Sian CTR3(KTR_PMAP, "region: %#zx - %#zx (%#zx)", 708265998Sian regions[i].mr_start, regions[i].mr_start + 709265998Sian regions[i].mr_size, regions[i].mr_size); 710190681Snwhitehorn if (hwphyssz != 0 && 711190681Snwhitehorn (physsz + regions[i].mr_size) >= hwphyssz) { 712190681Snwhitehorn if (physsz < hwphyssz) { 713190681Snwhitehorn phys_avail[j] = regions[i].mr_start; 714190681Snwhitehorn phys_avail[j + 1] = regions[i].mr_start + 715190681Snwhitehorn hwphyssz - physsz; 716190681Snwhitehorn physsz = hwphyssz; 717190681Snwhitehorn phys_avail_count++; 718190681Snwhitehorn } 719190681Snwhitehorn break; 720190681Snwhitehorn } 721190681Snwhitehorn phys_avail[j] = regions[i].mr_start; 722190681Snwhitehorn phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 723190681Snwhitehorn phys_avail_count++; 724190681Snwhitehorn physsz += regions[i].mr_size; 725190681Snwhitehorn } 726209975Snwhitehorn 727209975Snwhitehorn /* Check for overlap with the kernel and exception vectors */ 728209975Snwhitehorn for (j = 0; j < 2*phys_avail_count; j+=2) { 729209975Snwhitehorn if (phys_avail[j] < EXC_LAST) 730209975Snwhitehorn phys_avail[j] += EXC_LAST; 731209975Snwhitehorn 732209975Snwhitehorn if (kernelstart >= phys_avail[j] && 733209975Snwhitehorn kernelstart < phys_avail[j+1]) { 734209975Snwhitehorn if (kernelend < phys_avail[j+1]) { 735209975Snwhitehorn phys_avail[2*phys_avail_count] = 736209975Snwhitehorn (kernelend & ~PAGE_MASK) + PAGE_SIZE; 737209975Snwhitehorn phys_avail[2*phys_avail_count + 1] = 738209975Snwhitehorn phys_avail[j+1]; 739209975Snwhitehorn phys_avail_count++; 740209975Snwhitehorn } 741209975Snwhitehorn 742209975Snwhitehorn phys_avail[j+1] = kernelstart & ~PAGE_MASK; 743209975Snwhitehorn } 744209975Snwhitehorn 745209975Snwhitehorn if (kernelend >= phys_avail[j] && 746209975Snwhitehorn kernelend < phys_avail[j+1]) { 747209975Snwhitehorn if (kernelstart > phys_avail[j]) { 748209975Snwhitehorn phys_avail[2*phys_avail_count] = phys_avail[j]; 749209975Snwhitehorn phys_avail[2*phys_avail_count + 1] = 750209975Snwhitehorn kernelstart & ~PAGE_MASK; 751209975Snwhitehorn phys_avail_count++; 752209975Snwhitehorn } 753209975Snwhitehorn 754209975Snwhitehorn phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE; 755209975Snwhitehorn } 756209975Snwhitehorn } 757209975Snwhitehorn 758190681Snwhitehorn physmem = btoc(physsz); 759190681Snwhitehorn 760190681Snwhitehorn#ifdef PTEGCOUNT 761190681Snwhitehorn moea64_pteg_count = PTEGCOUNT; 762190681Snwhitehorn#else 763190681Snwhitehorn moea64_pteg_count = 0x1000; 764190681Snwhitehorn 765190681Snwhitehorn while (moea64_pteg_count < physmem) 766190681Snwhitehorn moea64_pteg_count <<= 1; 767209975Snwhitehorn 768209975Snwhitehorn moea64_pteg_count >>= 1; 769190681Snwhitehorn#endif /* PTEGCOUNT */ 770216174Snwhitehorn} 771190681Snwhitehorn 772216174Snwhitehornvoid 773216174Snwhitehornmoea64_mid_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 774216174Snwhitehorn{ 775216174Snwhitehorn vm_size_t size; 776216174Snwhitehorn register_t msr; 777216174Snwhitehorn int i; 778190681Snwhitehorn 779190681Snwhitehorn /* 780216174Snwhitehorn * Set PTEG mask 781190681Snwhitehorn */ 782190681Snwhitehorn moea64_pteg_mask = moea64_pteg_count - 1; 783190681Snwhitehorn 784190681Snwhitehorn /* 785190681Snwhitehorn * Allocate pv/overflow lists. 786190681Snwhitehorn */ 787190681Snwhitehorn size = sizeof(struct pvo_head) * moea64_pteg_count; 788190681Snwhitehorn 789190681Snwhitehorn moea64_pvo_table = (struct pvo_head *)moea64_bootstrap_alloc(size, 790190681Snwhitehorn PAGE_SIZE); 791190681Snwhitehorn CTR1(KTR_PMAP, "moea64_bootstrap: PVO table at %p", moea64_pvo_table); 792190681Snwhitehorn 793190681Snwhitehorn DISABLE_TRANS(msr); 794190681Snwhitehorn for (i = 0; i < moea64_pteg_count; i++) 795190681Snwhitehorn LIST_INIT(&moea64_pvo_table[i]); 796190681Snwhitehorn ENABLE_TRANS(msr); 797190681Snwhitehorn 798190681Snwhitehorn /* 799190681Snwhitehorn * Initialize the lock that synchronizes access to the pteg and pvo 800190681Snwhitehorn * tables. 801190681Snwhitehorn */ 802233529Snwhitehorn rw_init_flags(&moea64_table_lock, "pmap tables", RW_RECURSE); 803211967Snwhitehorn mtx_init(&moea64_slb_mutex, "SLB table", NULL, MTX_DEF); 804190681Snwhitehorn 805190681Snwhitehorn /* 806190681Snwhitehorn * Initialise the unmanaged pvo pool. 807190681Snwhitehorn */ 808190681Snwhitehorn moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc( 809190681Snwhitehorn BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); 810190681Snwhitehorn moea64_bpvo_pool_index = 0; 811190681Snwhitehorn 812190681Snwhitehorn /* 813190681Snwhitehorn * Make sure kernel vsid is allocated as well as VSID 0. 814190681Snwhitehorn */ 815209975Snwhitehorn #ifndef __powerpc64__ 816209975Snwhitehorn moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NVSIDS - 1)) / VSID_NBPW] 817190681Snwhitehorn |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 818190681Snwhitehorn moea64_vsid_bitmap[0] |= 1; 819209975Snwhitehorn #endif 820190681Snwhitehorn 821190681Snwhitehorn /* 822190681Snwhitehorn * Initialize the kernel pmap (which is statically allocated). 823190681Snwhitehorn */ 824209975Snwhitehorn #ifdef __powerpc64__ 825209975Snwhitehorn for (i = 0; i < 64; i++) { 826209975Snwhitehorn pcpup->pc_slb[i].slbv = 0; 827209975Snwhitehorn pcpup->pc_slb[i].slbe = 0; 828209975Snwhitehorn } 829209975Snwhitehorn #else 830190681Snwhitehorn for (i = 0; i < 16; i++) 831190681Snwhitehorn kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i; 832209975Snwhitehorn #endif 833190681Snwhitehorn 834190681Snwhitehorn kernel_pmap->pmap_phys = kernel_pmap; 835222813Sattilio CPU_FILL(&kernel_pmap->pm_active); 836235689Snwhitehorn RB_INIT(&kernel_pmap->pmap_pvo); 837190681Snwhitehorn 838190681Snwhitehorn PMAP_LOCK_INIT(kernel_pmap); 839190681Snwhitehorn 840190681Snwhitehorn /* 841190681Snwhitehorn * Now map in all the other buffers we allocated earlier 842190681Snwhitehorn */ 843190681Snwhitehorn 844209975Snwhitehorn moea64_setup_direct_map(mmup, kernelstart, kernelend); 845216174Snwhitehorn} 846190681Snwhitehorn 847216174Snwhitehornvoid 848216174Snwhitehornmoea64_late_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 849216174Snwhitehorn{ 850216174Snwhitehorn ihandle_t mmui; 851216174Snwhitehorn phandle_t chosen; 852216174Snwhitehorn phandle_t mmu; 853216174Snwhitehorn size_t sz; 854216174Snwhitehorn int i; 855216174Snwhitehorn vm_offset_t pa, va; 856216174Snwhitehorn void *dpcpu; 857216174Snwhitehorn 858190681Snwhitehorn /* 859209975Snwhitehorn * Set up the Open Firmware pmap and add its mappings if not in real 860209975Snwhitehorn * mode. 861190681Snwhitehorn */ 862190681Snwhitehorn 863215067Snwhitehorn chosen = OF_finddevice("/chosen"); 864215067Snwhitehorn if (chosen != -1 && OF_getprop(chosen, "mmu", &mmui, 4) != -1) { 865215158Snwhitehorn mmu = OF_instance_to_package(mmui); 866215158Snwhitehorn if (mmu == -1 || (sz = OF_getproplen(mmu, "translations")) == -1) 867215158Snwhitehorn sz = 0; 868199226Snwhitehorn if (sz > 6144 /* tmpstksz - 2 KB headroom */) 869199226Snwhitehorn panic("moea64_bootstrap: too many ofw translations"); 870190681Snwhitehorn 871215158Snwhitehorn if (sz > 0) 872215158Snwhitehorn moea64_add_ofw_mappings(mmup, mmu, sz); 873190681Snwhitehorn } 874190681Snwhitehorn 875190681Snwhitehorn /* 876190681Snwhitehorn * Calculate the last available physical address. 877190681Snwhitehorn */ 878190681Snwhitehorn for (i = 0; phys_avail[i + 2] != 0; i += 2) 879190681Snwhitehorn ; 880190681Snwhitehorn Maxmem = powerpc_btop(phys_avail[i + 1]); 881190681Snwhitehorn 882190681Snwhitehorn /* 883190681Snwhitehorn * Initialize MMU and remap early physical mappings 884190681Snwhitehorn */ 885216174Snwhitehorn MMU_CPU_BOOTSTRAP(mmup,0); 886222614Snwhitehorn mtmsr(mfmsr() | PSL_DR | PSL_IR); 887190681Snwhitehorn pmap_bootstrapped++; 888190681Snwhitehorn bs_remap_earlyboot(); 889190681Snwhitehorn 890190681Snwhitehorn /* 891190681Snwhitehorn * Set the start and end of kva. 892190681Snwhitehorn */ 893190681Snwhitehorn virtual_avail = VM_MIN_KERNEL_ADDRESS; 894204128Snwhitehorn virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 895190681Snwhitehorn 896190681Snwhitehorn /* 897209975Snwhitehorn * Map the entire KVA range into the SLB. We must not fault there. 898209975Snwhitehorn */ 899209975Snwhitehorn #ifdef __powerpc64__ 900209975Snwhitehorn for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH) 901209975Snwhitehorn moea64_bootstrap_slb_prefault(va, 0); 902209975Snwhitehorn #endif 903209975Snwhitehorn 904209975Snwhitehorn /* 905204128Snwhitehorn * Figure out how far we can extend virtual_end into segment 16 906204128Snwhitehorn * without running into existing mappings. Segment 16 is guaranteed 907204128Snwhitehorn * to contain neither RAM nor devices (at least on Apple hardware), 908204128Snwhitehorn * but will generally contain some OFW mappings we should not 909204128Snwhitehorn * step on. 910190681Snwhitehorn */ 911190681Snwhitehorn 912209975Snwhitehorn #ifndef __powerpc64__ /* KVA is in high memory on PPC64 */ 913204128Snwhitehorn PMAP_LOCK(kernel_pmap); 914209975Snwhitehorn while (virtual_end < VM_MAX_KERNEL_ADDRESS && 915209975Snwhitehorn moea64_pvo_find_va(kernel_pmap, virtual_end+1) == NULL) 916204128Snwhitehorn virtual_end += PAGE_SIZE; 917204128Snwhitehorn PMAP_UNLOCK(kernel_pmap); 918209975Snwhitehorn #endif 919190681Snwhitehorn 920190681Snwhitehorn /* 921190681Snwhitehorn * Allocate a kernel stack with a guard page for thread0 and map it 922190681Snwhitehorn * into the kernel page map. 923190681Snwhitehorn */ 924190681Snwhitehorn pa = moea64_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, PAGE_SIZE); 925190681Snwhitehorn va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 926190681Snwhitehorn virtual_avail = va + KSTACK_PAGES * PAGE_SIZE; 927220642Sandreast CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 928190681Snwhitehorn thread0.td_kstack = va; 929190681Snwhitehorn thread0.td_kstack_pages = KSTACK_PAGES; 930190681Snwhitehorn for (i = 0; i < KSTACK_PAGES; i++) { 931201758Smbr moea64_kenter(mmup, va, pa); 932190681Snwhitehorn pa += PAGE_SIZE; 933190681Snwhitehorn va += PAGE_SIZE; 934190681Snwhitehorn } 935190681Snwhitehorn 936190681Snwhitehorn /* 937190681Snwhitehorn * Allocate virtual address space for the message buffer. 938190681Snwhitehorn */ 939217688Spluknet pa = msgbuf_phys = moea64_bootstrap_alloc(msgbufsize, PAGE_SIZE); 940204297Snwhitehorn msgbufp = (struct msgbuf *)virtual_avail; 941204297Snwhitehorn va = virtual_avail; 942217688Spluknet virtual_avail += round_page(msgbufsize); 943204297Snwhitehorn while (va < virtual_avail) { 944204297Snwhitehorn moea64_kenter(mmup, va, pa); 945190681Snwhitehorn pa += PAGE_SIZE; 946204297Snwhitehorn va += PAGE_SIZE; 947190681Snwhitehorn } 948194784Sjeff 949194784Sjeff /* 950194784Sjeff * Allocate virtual address space for the dynamic percpu area. 951194784Sjeff */ 952194784Sjeff pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE); 953204297Snwhitehorn dpcpu = (void *)virtual_avail; 954209975Snwhitehorn va = virtual_avail; 955204297Snwhitehorn virtual_avail += DPCPU_SIZE; 956204297Snwhitehorn while (va < virtual_avail) { 957204297Snwhitehorn moea64_kenter(mmup, va, pa); 958194784Sjeff pa += PAGE_SIZE; 959204297Snwhitehorn va += PAGE_SIZE; 960194784Sjeff } 961194784Sjeff dpcpu_init(dpcpu, 0); 962216174Snwhitehorn 963216174Snwhitehorn /* 964216174Snwhitehorn * Allocate some things for page zeroing. We put this directly 965216174Snwhitehorn * in the page table, marked with LPTE_LOCKED, to avoid any 966216174Snwhitehorn * of the PVO book-keeping or other parts of the VM system 967216174Snwhitehorn * from even knowing that this hack exists. 968216174Snwhitehorn */ 969216174Snwhitehorn 970216174Snwhitehorn if (!hw_direct_map) { 971216174Snwhitehorn mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, 972216174Snwhitehorn MTX_DEF); 973216174Snwhitehorn for (i = 0; i < 2; i++) { 974216174Snwhitehorn moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE; 975216174Snwhitehorn virtual_end -= PAGE_SIZE; 976216174Snwhitehorn 977216174Snwhitehorn moea64_kenter(mmup, moea64_scratchpage_va[i], 0); 978216174Snwhitehorn 979216174Snwhitehorn moea64_scratchpage_pvo[i] = moea64_pvo_find_va( 980216174Snwhitehorn kernel_pmap, (vm_offset_t)moea64_scratchpage_va[i]); 981233529Snwhitehorn LOCK_TABLE_RD(); 982216174Snwhitehorn moea64_scratchpage_pte[i] = MOEA64_PVO_TO_PTE( 983216174Snwhitehorn mmup, moea64_scratchpage_pvo[i]); 984216174Snwhitehorn moea64_scratchpage_pvo[i]->pvo_pte.lpte.pte_hi 985216174Snwhitehorn |= LPTE_LOCKED; 986216174Snwhitehorn MOEA64_PTE_CHANGE(mmup, moea64_scratchpage_pte[i], 987216174Snwhitehorn &moea64_scratchpage_pvo[i]->pvo_pte.lpte, 988216174Snwhitehorn moea64_scratchpage_pvo[i]->pvo_vpn); 989233529Snwhitehorn UNLOCK_TABLE_RD(); 990216174Snwhitehorn } 991216174Snwhitehorn } 992190681Snwhitehorn} 993190681Snwhitehorn 994190681Snwhitehorn/* 995209975Snwhitehorn * Activate a user pmap. The pmap must be activated before its address 996190681Snwhitehorn * space can be accessed in any way. 997190681Snwhitehorn */ 998190681Snwhitehornvoid 999190681Snwhitehornmoea64_activate(mmu_t mmu, struct thread *td) 1000190681Snwhitehorn{ 1001209975Snwhitehorn pmap_t pm; 1002190681Snwhitehorn 1003190681Snwhitehorn pm = &td->td_proc->p_vmspace->vm_pmap; 1004223758Sattilio CPU_SET(PCPU_GET(cpuid), &pm->pm_active); 1005190681Snwhitehorn 1006209975Snwhitehorn #ifdef __powerpc64__ 1007209975Snwhitehorn PCPU_SET(userslb, pm->pm_slb); 1008209975Snwhitehorn #else 1009209975Snwhitehorn PCPU_SET(curpmap, pm->pmap_phys); 1010209975Snwhitehorn #endif 1011190681Snwhitehorn} 1012190681Snwhitehorn 1013190681Snwhitehornvoid 1014190681Snwhitehornmoea64_deactivate(mmu_t mmu, struct thread *td) 1015190681Snwhitehorn{ 1016190681Snwhitehorn pmap_t pm; 1017190681Snwhitehorn 1018190681Snwhitehorn pm = &td->td_proc->p_vmspace->vm_pmap; 1019223758Sattilio CPU_CLR(PCPU_GET(cpuid), &pm->pm_active); 1020209975Snwhitehorn #ifdef __powerpc64__ 1021209975Snwhitehorn PCPU_SET(userslb, NULL); 1022209975Snwhitehorn #else 1023190681Snwhitehorn PCPU_SET(curpmap, NULL); 1024209975Snwhitehorn #endif 1025190681Snwhitehorn} 1026190681Snwhitehorn 1027190681Snwhitehornvoid 1028270920Skibmoea64_unwire(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1029190681Snwhitehorn{ 1030270920Skib struct pvo_entry key, *pvo; 1031216174Snwhitehorn uintptr_t pt; 1032190681Snwhitehorn 1033270920Skib LOCK_TABLE_RD(); 1034190681Snwhitehorn PMAP_LOCK(pm); 1035270920Skib key.pvo_vaddr = sva; 1036270920Skib for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1037270920Skib pvo != NULL && PVO_VADDR(pvo) < eva; 1038270920Skib pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 1039216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 1040270920Skib if ((pvo->pvo_vaddr & PVO_WIRED) == 0) 1041270920Skib panic("moea64_unwire: pvo %p is missing PVO_WIRED", 1042270920Skib pvo); 1043270920Skib pvo->pvo_vaddr &= ~PVO_WIRED; 1044270920Skib if ((pvo->pvo_pte.lpte.pte_hi & LPTE_WIRED) == 0) 1045270920Skib panic("moea64_unwire: pte %p is missing LPTE_WIRED", 1046270920Skib &pvo->pvo_pte.lpte); 1047270920Skib pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED; 1048216174Snwhitehorn if (pt != -1) { 1049270920Skib /* 1050270920Skib * The PTE's wired attribute is not a hardware 1051270920Skib * feature, so there is no need to invalidate any TLB 1052270920Skib * entries. 1053270920Skib */ 1054216174Snwhitehorn MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte, 1055209975Snwhitehorn pvo->pvo_vpn); 1056209975Snwhitehorn } 1057270920Skib pm->pm_stats.wired_count--; 1058190681Snwhitehorn } 1059270920Skib UNLOCK_TABLE_RD(); 1060190681Snwhitehorn PMAP_UNLOCK(pm); 1061190681Snwhitehorn} 1062190681Snwhitehorn 1063190681Snwhitehorn/* 1064190681Snwhitehorn * This goes through and sets the physical address of our 1065190681Snwhitehorn * special scratch PTE to the PA we want to zero or copy. Because 1066190681Snwhitehorn * of locking issues (this can get called in pvo_enter() by 1067190681Snwhitehorn * the UMA allocator), we can't use most other utility functions here 1068190681Snwhitehorn */ 1069190681Snwhitehorn 1070190681Snwhitehornstatic __inline 1071216174Snwhitehornvoid moea64_set_scratchpage_pa(mmu_t mmup, int which, vm_offset_t pa) { 1072204694Snwhitehorn 1073209975Snwhitehorn KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!")); 1074204268Snwhitehorn mtx_assert(&moea64_scratchpage_mtx, MA_OWNED); 1075204268Snwhitehorn 1076216174Snwhitehorn moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo &= 1077204694Snwhitehorn ~(LPTE_WIMG | LPTE_RPGN); 1078216174Snwhitehorn moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo |= 1079213307Snwhitehorn moea64_calc_wimg(pa, VM_MEMATTR_DEFAULT) | (uint64_t)pa; 1080216174Snwhitehorn MOEA64_PTE_CHANGE(mmup, moea64_scratchpage_pte[which], 1081216174Snwhitehorn &moea64_scratchpage_pvo[which]->pvo_pte.lpte, 1082216174Snwhitehorn moea64_scratchpage_pvo[which]->pvo_vpn); 1083216383Snwhitehorn isync(); 1084190681Snwhitehorn} 1085190681Snwhitehorn 1086190681Snwhitehornvoid 1087190681Snwhitehornmoea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) 1088190681Snwhitehorn{ 1089190681Snwhitehorn vm_offset_t dst; 1090190681Snwhitehorn vm_offset_t src; 1091190681Snwhitehorn 1092190681Snwhitehorn dst = VM_PAGE_TO_PHYS(mdst); 1093190681Snwhitehorn src = VM_PAGE_TO_PHYS(msrc); 1094190681Snwhitehorn 1095209975Snwhitehorn if (hw_direct_map) { 1096234156Snwhitehorn bcopy((void *)src, (void *)dst, PAGE_SIZE); 1097209975Snwhitehorn } else { 1098209975Snwhitehorn mtx_lock(&moea64_scratchpage_mtx); 1099190681Snwhitehorn 1100216174Snwhitehorn moea64_set_scratchpage_pa(mmu, 0, src); 1101216174Snwhitehorn moea64_set_scratchpage_pa(mmu, 1, dst); 1102190681Snwhitehorn 1103234156Snwhitehorn bcopy((void *)moea64_scratchpage_va[0], 1104209975Snwhitehorn (void *)moea64_scratchpage_va[1], PAGE_SIZE); 1105190681Snwhitehorn 1106209975Snwhitehorn mtx_unlock(&moea64_scratchpage_mtx); 1107209975Snwhitehorn } 1108190681Snwhitehorn} 1109190681Snwhitehorn 1110248280Skibstatic inline void 1111248280Skibmoea64_copy_pages_dmap(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 1112248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize) 1113248280Skib{ 1114248280Skib void *a_cp, *b_cp; 1115248280Skib vm_offset_t a_pg_offset, b_pg_offset; 1116248280Skib int cnt; 1117248280Skib 1118248280Skib while (xfersize > 0) { 1119248280Skib a_pg_offset = a_offset & PAGE_MASK; 1120248280Skib cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 1121248280Skib a_cp = (char *)VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]) + 1122248280Skib a_pg_offset; 1123248280Skib b_pg_offset = b_offset & PAGE_MASK; 1124248280Skib cnt = min(cnt, PAGE_SIZE - b_pg_offset); 1125248280Skib b_cp = (char *)VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]) + 1126248280Skib b_pg_offset; 1127248280Skib bcopy(a_cp, b_cp, cnt); 1128248280Skib a_offset += cnt; 1129248280Skib b_offset += cnt; 1130248280Skib xfersize -= cnt; 1131248280Skib } 1132248280Skib} 1133248280Skib 1134248280Skibstatic inline void 1135248280Skibmoea64_copy_pages_nodmap(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 1136248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize) 1137248280Skib{ 1138248280Skib void *a_cp, *b_cp; 1139248280Skib vm_offset_t a_pg_offset, b_pg_offset; 1140248280Skib int cnt; 1141248280Skib 1142248280Skib mtx_lock(&moea64_scratchpage_mtx); 1143248280Skib while (xfersize > 0) { 1144248280Skib a_pg_offset = a_offset & PAGE_MASK; 1145248280Skib cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 1146248280Skib moea64_set_scratchpage_pa(mmu, 0, 1147248280Skib VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])); 1148248280Skib a_cp = (char *)moea64_scratchpage_va[0] + a_pg_offset; 1149248280Skib b_pg_offset = b_offset & PAGE_MASK; 1150248280Skib cnt = min(cnt, PAGE_SIZE - b_pg_offset); 1151248280Skib moea64_set_scratchpage_pa(mmu, 1, 1152248280Skib VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])); 1153248280Skib b_cp = (char *)moea64_scratchpage_va[1] + b_pg_offset; 1154248280Skib bcopy(a_cp, b_cp, cnt); 1155248280Skib a_offset += cnt; 1156248280Skib b_offset += cnt; 1157248280Skib xfersize -= cnt; 1158248280Skib } 1159248280Skib mtx_unlock(&moea64_scratchpage_mtx); 1160248280Skib} 1161248280Skib 1162190681Snwhitehornvoid 1163248280Skibmoea64_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 1164248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize) 1165248280Skib{ 1166248280Skib 1167248280Skib if (hw_direct_map) { 1168248280Skib moea64_copy_pages_dmap(mmu, ma, a_offset, mb, b_offset, 1169248280Skib xfersize); 1170248280Skib } else { 1171248280Skib moea64_copy_pages_nodmap(mmu, ma, a_offset, mb, b_offset, 1172248280Skib xfersize); 1173248280Skib } 1174248280Skib} 1175248280Skib 1176248280Skibvoid 1177190681Snwhitehornmoea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) 1178190681Snwhitehorn{ 1179190681Snwhitehorn vm_offset_t pa = VM_PAGE_TO_PHYS(m); 1180190681Snwhitehorn 1181190681Snwhitehorn if (size + off > PAGE_SIZE) 1182190681Snwhitehorn panic("moea64_zero_page: size + off > PAGE_SIZE"); 1183190681Snwhitehorn 1184209975Snwhitehorn if (hw_direct_map) { 1185209975Snwhitehorn bzero((caddr_t)pa + off, size); 1186209975Snwhitehorn } else { 1187209975Snwhitehorn mtx_lock(&moea64_scratchpage_mtx); 1188216174Snwhitehorn moea64_set_scratchpage_pa(mmu, 0, pa); 1189209975Snwhitehorn bzero((caddr_t)moea64_scratchpage_va[0] + off, size); 1190209975Snwhitehorn mtx_unlock(&moea64_scratchpage_mtx); 1191209975Snwhitehorn } 1192190681Snwhitehorn} 1193190681Snwhitehorn 1194204269Snwhitehorn/* 1195204269Snwhitehorn * Zero a page of physical memory by temporarily mapping it 1196204269Snwhitehorn */ 1197190681Snwhitehornvoid 1198204269Snwhitehornmoea64_zero_page(mmu_t mmu, vm_page_t m) 1199204269Snwhitehorn{ 1200204269Snwhitehorn vm_offset_t pa = VM_PAGE_TO_PHYS(m); 1201209975Snwhitehorn vm_offset_t va, off; 1202204269Snwhitehorn 1203209975Snwhitehorn if (!hw_direct_map) { 1204209975Snwhitehorn mtx_lock(&moea64_scratchpage_mtx); 1205204269Snwhitehorn 1206216174Snwhitehorn moea64_set_scratchpage_pa(mmu, 0, pa); 1207209975Snwhitehorn va = moea64_scratchpage_va[0]; 1208209975Snwhitehorn } else { 1209209975Snwhitehorn va = pa; 1210209975Snwhitehorn } 1211209975Snwhitehorn 1212204269Snwhitehorn for (off = 0; off < PAGE_SIZE; off += cacheline_size) 1213209975Snwhitehorn __asm __volatile("dcbz 0,%0" :: "r"(va + off)); 1214209975Snwhitehorn 1215209975Snwhitehorn if (!hw_direct_map) 1216209975Snwhitehorn mtx_unlock(&moea64_scratchpage_mtx); 1217204269Snwhitehorn} 1218204269Snwhitehorn 1219204269Snwhitehornvoid 1220190681Snwhitehornmoea64_zero_page_idle(mmu_t mmu, vm_page_t m) 1221190681Snwhitehorn{ 1222190681Snwhitehorn 1223190681Snwhitehorn moea64_zero_page(mmu, m); 1224190681Snwhitehorn} 1225190681Snwhitehorn 1226190681Snwhitehorn/* 1227190681Snwhitehorn * Map the given physical page at the specified virtual address in the 1228190681Snwhitehorn * target pmap with the protection requested. If specified the page 1229190681Snwhitehorn * will be wired down. 1230190681Snwhitehorn */ 1231233957Snwhitehorn 1232270439Skibint 1233190681Snwhitehornmoea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 1234270439Skib vm_prot_t prot, u_int flags, int8_t psind) 1235190681Snwhitehorn{ 1236190681Snwhitehorn struct pvo_head *pvo_head; 1237190681Snwhitehorn uma_zone_t zone; 1238190681Snwhitehorn vm_page_t pg; 1239190681Snwhitehorn uint64_t pte_lo; 1240190681Snwhitehorn u_int pvo_flags; 1241190681Snwhitehorn int error; 1242190681Snwhitehorn 1243190681Snwhitehorn if (!moea64_initialized) { 1244235689Snwhitehorn pvo_head = NULL; 1245190681Snwhitehorn pg = NULL; 1246190681Snwhitehorn zone = moea64_upvo_zone; 1247190681Snwhitehorn pvo_flags = 0; 1248190681Snwhitehorn } else { 1249190681Snwhitehorn pvo_head = vm_page_to_pvoh(m); 1250190681Snwhitehorn pg = m; 1251190681Snwhitehorn zone = moea64_mpvo_zone; 1252190681Snwhitehorn pvo_flags = PVO_MANAGED; 1253190681Snwhitehorn } 1254190681Snwhitehorn 1255254138Sattilio if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) 1256250747Salc VM_OBJECT_ASSERT_LOCKED(m->object); 1257190681Snwhitehorn 1258190681Snwhitehorn /* XXX change the pvo head for fake pages */ 1259224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) { 1260190681Snwhitehorn pvo_flags &= ~PVO_MANAGED; 1261235689Snwhitehorn pvo_head = NULL; 1262190681Snwhitehorn zone = moea64_upvo_zone; 1263190681Snwhitehorn } 1264190681Snwhitehorn 1265213307Snwhitehorn pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); 1266190681Snwhitehorn 1267190681Snwhitehorn if (prot & VM_PROT_WRITE) { 1268190681Snwhitehorn pte_lo |= LPTE_BW; 1269208810Salc if (pmap_bootstrapped && 1270224746Skib (m->oflags & VPO_UNMANAGED) == 0) 1271225418Skib vm_page_aflag_set(m, PGA_WRITEABLE); 1272190681Snwhitehorn } else 1273190681Snwhitehorn pte_lo |= LPTE_BR; 1274190681Snwhitehorn 1275217341Snwhitehorn if ((prot & VM_PROT_EXECUTE) == 0) 1276217341Snwhitehorn pte_lo |= LPTE_NOEXEC; 1277190681Snwhitehorn 1278270439Skib if ((flags & PMAP_ENTER_WIRED) != 0) 1279190681Snwhitehorn pvo_flags |= PVO_WIRED; 1280190681Snwhitehorn 1281270439Skib for (;;) { 1282270439Skib LOCK_TABLE_WR(); 1283270439Skib PMAP_LOCK(pmap); 1284270439Skib error = moea64_pvo_enter(mmu, pmap, zone, pvo_head, va, 1285270439Skib VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags, psind); 1286270439Skib PMAP_UNLOCK(pmap); 1287270439Skib UNLOCK_TABLE_WR(); 1288270439Skib if (error != ENOMEM) 1289270439Skib break; 1290270439Skib if ((flags & PMAP_ENTER_NOSLEEP) != 0) 1291270439Skib return (KERN_RESOURCE_SHORTAGE); 1292270439Skib VM_OBJECT_ASSERT_UNLOCKED(m->object); 1293270439Skib VM_WAIT; 1294270439Skib } 1295190681Snwhitehorn 1296190681Snwhitehorn /* 1297190681Snwhitehorn * Flush the page from the instruction cache if this page is 1298190681Snwhitehorn * mapped executable and cacheable. 1299190681Snwhitehorn */ 1300233949Snwhitehorn if (pmap != kernel_pmap && !(m->aflags & PGA_EXECUTABLE) && 1301233949Snwhitehorn (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { 1302233949Snwhitehorn vm_page_aflag_set(m, PGA_EXECUTABLE); 1303216174Snwhitehorn moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE); 1304233949Snwhitehorn } 1305270439Skib return (KERN_SUCCESS); 1306190681Snwhitehorn} 1307190681Snwhitehorn 1308190681Snwhitehornstatic void 1309216174Snwhitehornmoea64_syncicache(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_offset_t pa, 1310216174Snwhitehorn vm_size_t sz) 1311190681Snwhitehorn{ 1312204042Snwhitehorn 1313190681Snwhitehorn /* 1314190681Snwhitehorn * This is much trickier than on older systems because 1315190681Snwhitehorn * we can't sync the icache on physical addresses directly 1316190681Snwhitehorn * without a direct map. Instead we check a couple of cases 1317190681Snwhitehorn * where the memory is already mapped in and, failing that, 1318190681Snwhitehorn * use the same trick we use for page zeroing to create 1319190681Snwhitehorn * a temporary mapping for this physical address. 1320190681Snwhitehorn */ 1321190681Snwhitehorn 1322190681Snwhitehorn if (!pmap_bootstrapped) { 1323190681Snwhitehorn /* 1324190681Snwhitehorn * If PMAP is not bootstrapped, we are likely to be 1325190681Snwhitehorn * in real mode. 1326190681Snwhitehorn */ 1327198341Smarcel __syncicache((void *)pa, sz); 1328190681Snwhitehorn } else if (pmap == kernel_pmap) { 1329198341Smarcel __syncicache((void *)va, sz); 1330209975Snwhitehorn } else if (hw_direct_map) { 1331209975Snwhitehorn __syncicache((void *)pa, sz); 1332190681Snwhitehorn } else { 1333190681Snwhitehorn /* Use the scratch page to set up a temp mapping */ 1334190681Snwhitehorn 1335190681Snwhitehorn mtx_lock(&moea64_scratchpage_mtx); 1336190681Snwhitehorn 1337216174Snwhitehorn moea64_set_scratchpage_pa(mmu, 1, pa & ~ADDR_POFF); 1338204042Snwhitehorn __syncicache((void *)(moea64_scratchpage_va[1] + 1339204042Snwhitehorn (va & ADDR_POFF)), sz); 1340190681Snwhitehorn 1341190681Snwhitehorn mtx_unlock(&moea64_scratchpage_mtx); 1342190681Snwhitehorn } 1343190681Snwhitehorn} 1344190681Snwhitehorn 1345190681Snwhitehorn/* 1346190681Snwhitehorn * Maps a sequence of resident pages belonging to the same object. 1347190681Snwhitehorn * The sequence begins with the given page m_start. This page is 1348190681Snwhitehorn * mapped at the given virtual address start. Each subsequent page is 1349190681Snwhitehorn * mapped at a virtual address that is offset from start by the same 1350190681Snwhitehorn * amount as the page is offset from m_start within the object. The 1351190681Snwhitehorn * last page in the sequence is the page with the largest offset from 1352190681Snwhitehorn * m_start that can be mapped at a virtual address less than the given 1353190681Snwhitehorn * virtual address end. Not every virtual page between start and end 1354190681Snwhitehorn * is mapped; only those for which a resident page exists with the 1355190681Snwhitehorn * corresponding offset from m_start are mapped. 1356190681Snwhitehorn */ 1357190681Snwhitehornvoid 1358190681Snwhitehornmoea64_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, 1359190681Snwhitehorn vm_page_t m_start, vm_prot_t prot) 1360190681Snwhitehorn{ 1361190681Snwhitehorn vm_page_t m; 1362190681Snwhitehorn vm_pindex_t diff, psize; 1363190681Snwhitehorn 1364250884Sattilio VM_OBJECT_ASSERT_LOCKED(m_start->object); 1365250884Sattilio 1366190681Snwhitehorn psize = atop(end - start); 1367190681Snwhitehorn m = m_start; 1368190681Snwhitehorn while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1369233957Snwhitehorn moea64_enter(mmu, pm, start + ptoa(diff), m, prot & 1370270439Skib (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP, 0); 1371190681Snwhitehorn m = TAILQ_NEXT(m, listq); 1372190681Snwhitehorn } 1373190681Snwhitehorn} 1374190681Snwhitehorn 1375190681Snwhitehornvoid 1376190681Snwhitehornmoea64_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, 1377190681Snwhitehorn vm_prot_t prot) 1378190681Snwhitehorn{ 1379207796Salc 1380270439Skib moea64_enter(mmu, pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1381270439Skib PMAP_ENTER_NOSLEEP, 0); 1382190681Snwhitehorn} 1383190681Snwhitehorn 1384190681Snwhitehornvm_paddr_t 1385190681Snwhitehornmoea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) 1386190681Snwhitehorn{ 1387190681Snwhitehorn struct pvo_entry *pvo; 1388190681Snwhitehorn vm_paddr_t pa; 1389190681Snwhitehorn 1390190681Snwhitehorn PMAP_LOCK(pm); 1391209975Snwhitehorn pvo = moea64_pvo_find_va(pm, va); 1392190681Snwhitehorn if (pvo == NULL) 1393190681Snwhitehorn pa = 0; 1394190681Snwhitehorn else 1395209975Snwhitehorn pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | 1396209975Snwhitehorn (va - PVO_VADDR(pvo)); 1397190681Snwhitehorn PMAP_UNLOCK(pm); 1398190681Snwhitehorn return (pa); 1399190681Snwhitehorn} 1400190681Snwhitehorn 1401190681Snwhitehorn/* 1402190681Snwhitehorn * Atomically extract and hold the physical page with the given 1403190681Snwhitehorn * pmap and virtual address pair if that mapping permits the given 1404190681Snwhitehorn * protection. 1405190681Snwhitehorn */ 1406190681Snwhitehornvm_page_t 1407190681Snwhitehornmoea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) 1408190681Snwhitehorn{ 1409190681Snwhitehorn struct pvo_entry *pvo; 1410190681Snwhitehorn vm_page_t m; 1411207410Skmacy vm_paddr_t pa; 1412190681Snwhitehorn 1413190681Snwhitehorn m = NULL; 1414207410Skmacy pa = 0; 1415190681Snwhitehorn PMAP_LOCK(pmap); 1416207410Skmacyretry: 1417209975Snwhitehorn pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); 1418190681Snwhitehorn if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 1419190681Snwhitehorn ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW || 1420190681Snwhitehorn (prot & VM_PROT_WRITE) == 0)) { 1421235689Snwhitehorn if (vm_page_pa_tryrelock(pmap, 1422207410Skmacy pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, &pa)) 1423207410Skmacy goto retry; 1424190681Snwhitehorn m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); 1425190681Snwhitehorn vm_page_hold(m); 1426190681Snwhitehorn } 1427207410Skmacy PA_UNLOCK_COND(pa); 1428190681Snwhitehorn PMAP_UNLOCK(pmap); 1429190681Snwhitehorn return (m); 1430190681Snwhitehorn} 1431190681Snwhitehorn 1432216174Snwhitehornstatic mmu_t installed_mmu; 1433216174Snwhitehorn 1434190681Snwhitehornstatic void * 1435190681Snwhitehornmoea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 1436190681Snwhitehorn{ 1437190681Snwhitehorn /* 1438190681Snwhitehorn * This entire routine is a horrible hack to avoid bothering kmem 1439190681Snwhitehorn * for new KVA addresses. Because this can get called from inside 1440190681Snwhitehorn * kmem allocation routines, calling kmem for a new address here 1441190681Snwhitehorn * can lead to multiply locking non-recursive mutexes. 1442190681Snwhitehorn */ 1443190681Snwhitehorn vm_offset_t va; 1444190681Snwhitehorn 1445190681Snwhitehorn vm_page_t m; 1446190681Snwhitehorn int pflags, needed_lock; 1447190681Snwhitehorn 1448190681Snwhitehorn *flags = UMA_SLAB_PRIV; 1449190681Snwhitehorn needed_lock = !PMAP_LOCKED(kernel_pmap); 1450243040Skib pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; 1451190681Snwhitehorn 1452190681Snwhitehorn for (;;) { 1453228522Salc m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ); 1454190681Snwhitehorn if (m == NULL) { 1455190681Snwhitehorn if (wait & M_NOWAIT) 1456190681Snwhitehorn return (NULL); 1457190681Snwhitehorn VM_WAIT; 1458190681Snwhitehorn } else 1459190681Snwhitehorn break; 1460190681Snwhitehorn } 1461190681Snwhitehorn 1462204128Snwhitehorn va = VM_PAGE_TO_PHYS(m); 1463190681Snwhitehorn 1464233529Snwhitehorn LOCK_TABLE_WR(); 1465233529Snwhitehorn if (needed_lock) 1466233529Snwhitehorn PMAP_LOCK(kernel_pmap); 1467233529Snwhitehorn 1468216174Snwhitehorn moea64_pvo_enter(installed_mmu, kernel_pmap, moea64_upvo_zone, 1469270439Skib NULL, va, VM_PAGE_TO_PHYS(m), LPTE_M, PVO_WIRED | PVO_BOOTSTRAP, 1470270439Skib 0); 1471190681Snwhitehorn 1472190681Snwhitehorn if (needed_lock) 1473190681Snwhitehorn PMAP_UNLOCK(kernel_pmap); 1474233529Snwhitehorn UNLOCK_TABLE_WR(); 1475198378Snwhitehorn 1476190681Snwhitehorn if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) 1477190681Snwhitehorn bzero((void *)va, PAGE_SIZE); 1478190681Snwhitehorn 1479190681Snwhitehorn return (void *)va; 1480190681Snwhitehorn} 1481190681Snwhitehorn 1482230767Skibextern int elf32_nxstack; 1483230767Skib 1484190681Snwhitehornvoid 1485190681Snwhitehornmoea64_init(mmu_t mmu) 1486190681Snwhitehorn{ 1487190681Snwhitehorn 1488190681Snwhitehorn CTR0(KTR_PMAP, "moea64_init"); 1489190681Snwhitehorn 1490190681Snwhitehorn moea64_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), 1491190681Snwhitehorn NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1492190681Snwhitehorn UMA_ZONE_VM | UMA_ZONE_NOFREE); 1493190681Snwhitehorn moea64_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), 1494190681Snwhitehorn NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1495190681Snwhitehorn UMA_ZONE_VM | UMA_ZONE_NOFREE); 1496190681Snwhitehorn 1497190681Snwhitehorn if (!hw_direct_map) { 1498216174Snwhitehorn installed_mmu = mmu; 1499190681Snwhitehorn uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc); 1500190681Snwhitehorn uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc); 1501190681Snwhitehorn } 1502190681Snwhitehorn 1503230779Skib#ifdef COMPAT_FREEBSD32 1504230767Skib elf32_nxstack = 1; 1505230779Skib#endif 1506230767Skib 1507190681Snwhitehorn moea64_initialized = TRUE; 1508190681Snwhitehorn} 1509190681Snwhitehorn 1510190681Snwhitehornboolean_t 1511207155Salcmoea64_is_referenced(mmu_t mmu, vm_page_t m) 1512207155Salc{ 1513207155Salc 1514224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1515208574Salc ("moea64_is_referenced: page %p is not managed", m)); 1516216174Snwhitehorn return (moea64_query_bit(mmu, m, PTE_REF)); 1517207155Salc} 1518207155Salc 1519207155Salcboolean_t 1520190681Snwhitehornmoea64_is_modified(mmu_t mmu, vm_page_t m) 1521190681Snwhitehorn{ 1522190681Snwhitehorn 1523224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1524208504Salc ("moea64_is_modified: page %p is not managed", m)); 1525208504Salc 1526208504Salc /* 1527254138Sattilio * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 1528225418Skib * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 1529208504Salc * is clear, no PTEs can have LPTE_CHG set. 1530208504Salc */ 1531255503Snwhitehorn VM_OBJECT_ASSERT_LOCKED(m->object); 1532254138Sattilio if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 1533190681Snwhitehorn return (FALSE); 1534216174Snwhitehorn return (moea64_query_bit(mmu, m, LPTE_CHG)); 1535190681Snwhitehorn} 1536190681Snwhitehorn 1537214617Salcboolean_t 1538214617Salcmoea64_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t va) 1539214617Salc{ 1540214617Salc struct pvo_entry *pvo; 1541214617Salc boolean_t rv; 1542214617Salc 1543214617Salc PMAP_LOCK(pmap); 1544214617Salc pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); 1545214617Salc rv = pvo == NULL || (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0; 1546214617Salc PMAP_UNLOCK(pmap); 1547214617Salc return (rv); 1548214617Salc} 1549214617Salc 1550190681Snwhitehornvoid 1551190681Snwhitehornmoea64_clear_modify(mmu_t mmu, vm_page_t m) 1552190681Snwhitehorn{ 1553190681Snwhitehorn 1554224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1555208504Salc ("moea64_clear_modify: page %p is not managed", m)); 1556248084Sattilio VM_OBJECT_ASSERT_WLOCKED(m->object); 1557254138Sattilio KASSERT(!vm_page_xbusied(m), 1558254138Sattilio ("moea64_clear_modify: page %p is exclusive busied", m)); 1559208504Salc 1560208504Salc /* 1561225418Skib * If the page is not PGA_WRITEABLE, then no PTEs can have LPTE_CHG 1562208504Salc * set. If the object containing the page is locked and the page is 1563254138Sattilio * not exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 1564208504Salc */ 1565225418Skib if ((m->aflags & PGA_WRITEABLE) == 0) 1566190681Snwhitehorn return; 1567216174Snwhitehorn moea64_clear_bit(mmu, m, LPTE_CHG); 1568190681Snwhitehorn} 1569190681Snwhitehorn 1570190681Snwhitehorn/* 1571190681Snwhitehorn * Clear the write and modified bits in each of the given page's mappings. 1572190681Snwhitehorn */ 1573190681Snwhitehornvoid 1574190681Snwhitehornmoea64_remove_write(mmu_t mmu, vm_page_t m) 1575190681Snwhitehorn{ 1576190681Snwhitehorn struct pvo_entry *pvo; 1577216174Snwhitehorn uintptr_t pt; 1578190681Snwhitehorn pmap_t pmap; 1579233434Snwhitehorn uint64_t lo = 0; 1580190681Snwhitehorn 1581224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1582208175Salc ("moea64_remove_write: page %p is not managed", m)); 1583208175Salc 1584208175Salc /* 1585254138Sattilio * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 1586254138Sattilio * set by another thread while the object is locked. Thus, 1587254138Sattilio * if PGA_WRITEABLE is clear, no page table entries need updating. 1588208175Salc */ 1589248084Sattilio VM_OBJECT_ASSERT_WLOCKED(m->object); 1590254138Sattilio if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 1591190681Snwhitehorn return; 1592216174Snwhitehorn powerpc_sync(); 1593233529Snwhitehorn LOCK_TABLE_RD(); 1594190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1595190681Snwhitehorn pmap = pvo->pvo_pmap; 1596190681Snwhitehorn PMAP_LOCK(pmap); 1597190681Snwhitehorn if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { 1598216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 1599190681Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP; 1600190681Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= LPTE_BR; 1601216174Snwhitehorn if (pt != -1) { 1602216174Snwhitehorn MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte); 1603190681Snwhitehorn lo |= pvo->pvo_pte.lpte.pte_lo; 1604190681Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG; 1605216174Snwhitehorn MOEA64_PTE_CHANGE(mmu, pt, 1606216174Snwhitehorn &pvo->pvo_pte.lpte, pvo->pvo_vpn); 1607209975Snwhitehorn if (pvo->pvo_pmap == kernel_pmap) 1608209975Snwhitehorn isync(); 1609190681Snwhitehorn } 1610190681Snwhitehorn } 1611233530Snwhitehorn if ((lo & LPTE_CHG) != 0) 1612233530Snwhitehorn vm_page_dirty(m); 1613190681Snwhitehorn PMAP_UNLOCK(pmap); 1614190681Snwhitehorn } 1615233529Snwhitehorn UNLOCK_TABLE_RD(); 1616225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1617190681Snwhitehorn} 1618190681Snwhitehorn 1619190681Snwhitehorn/* 1620190681Snwhitehorn * moea64_ts_referenced: 1621190681Snwhitehorn * 1622190681Snwhitehorn * Return a count of reference bits for a page, clearing those bits. 1623190681Snwhitehorn * It is not necessary for every reference bit to be cleared, but it 1624190681Snwhitehorn * is necessary that 0 only be returned when there are truly no 1625190681Snwhitehorn * reference bits set. 1626190681Snwhitehorn * 1627190681Snwhitehorn * XXX: The exact number of bits to check and clear is a matter that 1628190681Snwhitehorn * should be tested and standardized at some point in the future for 1629190681Snwhitehorn * optimal aging of shared pages. 1630190681Snwhitehorn */ 1631238357Salcint 1632190681Snwhitehornmoea64_ts_referenced(mmu_t mmu, vm_page_t m) 1633190681Snwhitehorn{ 1634190681Snwhitehorn 1635224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1636208990Salc ("moea64_ts_referenced: page %p is not managed", m)); 1637216174Snwhitehorn return (moea64_clear_bit(mmu, m, LPTE_REF)); 1638190681Snwhitehorn} 1639190681Snwhitehorn 1640190681Snwhitehorn/* 1641213307Snwhitehorn * Modify the WIMG settings of all mappings for a page. 1642213307Snwhitehorn */ 1643213307Snwhitehornvoid 1644213307Snwhitehornmoea64_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma) 1645213307Snwhitehorn{ 1646213307Snwhitehorn struct pvo_entry *pvo; 1647213335Snwhitehorn struct pvo_head *pvo_head; 1648216174Snwhitehorn uintptr_t pt; 1649213307Snwhitehorn pmap_t pmap; 1650213307Snwhitehorn uint64_t lo; 1651213307Snwhitehorn 1652224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) { 1653213335Snwhitehorn m->md.mdpg_cache_attrs = ma; 1654213335Snwhitehorn return; 1655213335Snwhitehorn } 1656213335Snwhitehorn 1657213335Snwhitehorn pvo_head = vm_page_to_pvoh(m); 1658213307Snwhitehorn lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma); 1659233529Snwhitehorn LOCK_TABLE_RD(); 1660213335Snwhitehorn LIST_FOREACH(pvo, pvo_head, pvo_vlink) { 1661213307Snwhitehorn pmap = pvo->pvo_pmap; 1662213307Snwhitehorn PMAP_LOCK(pmap); 1663216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 1664213307Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~LPTE_WIMG; 1665213307Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= lo; 1666216174Snwhitehorn if (pt != -1) { 1667216174Snwhitehorn MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte, 1668213307Snwhitehorn pvo->pvo_vpn); 1669213307Snwhitehorn if (pvo->pvo_pmap == kernel_pmap) 1670213307Snwhitehorn isync(); 1671213307Snwhitehorn } 1672213307Snwhitehorn PMAP_UNLOCK(pmap); 1673213307Snwhitehorn } 1674233529Snwhitehorn UNLOCK_TABLE_RD(); 1675213307Snwhitehorn m->md.mdpg_cache_attrs = ma; 1676213307Snwhitehorn} 1677213307Snwhitehorn 1678213307Snwhitehorn/* 1679190681Snwhitehorn * Map a wired page into kernel virtual address space. 1680190681Snwhitehorn */ 1681190681Snwhitehornvoid 1682213307Snwhitehornmoea64_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma) 1683190681Snwhitehorn{ 1684190681Snwhitehorn uint64_t pte_lo; 1685190681Snwhitehorn int error; 1686190681Snwhitehorn 1687213307Snwhitehorn pte_lo = moea64_calc_wimg(pa, ma); 1688190681Snwhitehorn 1689233529Snwhitehorn LOCK_TABLE_WR(); 1690190681Snwhitehorn PMAP_LOCK(kernel_pmap); 1691216174Snwhitehorn error = moea64_pvo_enter(mmu, kernel_pmap, moea64_upvo_zone, 1692270439Skib NULL, va, pa, pte_lo, PVO_WIRED, 0); 1693233529Snwhitehorn PMAP_UNLOCK(kernel_pmap); 1694233529Snwhitehorn UNLOCK_TABLE_WR(); 1695190681Snwhitehorn 1696190681Snwhitehorn if (error != 0 && error != ENOENT) 1697209975Snwhitehorn panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va, 1698190681Snwhitehorn pa, error); 1699190681Snwhitehorn} 1700190681Snwhitehorn 1701213307Snwhitehornvoid 1702236019Srajmoea64_kenter(mmu_t mmu, vm_offset_t va, vm_paddr_t pa) 1703213307Snwhitehorn{ 1704213307Snwhitehorn 1705213307Snwhitehorn moea64_kenter_attr(mmu, va, pa, VM_MEMATTR_DEFAULT); 1706213307Snwhitehorn} 1707213307Snwhitehorn 1708190681Snwhitehorn/* 1709190681Snwhitehorn * Extract the physical page address associated with the given kernel virtual 1710190681Snwhitehorn * address. 1711190681Snwhitehorn */ 1712236019Srajvm_paddr_t 1713190681Snwhitehornmoea64_kextract(mmu_t mmu, vm_offset_t va) 1714190681Snwhitehorn{ 1715190681Snwhitehorn struct pvo_entry *pvo; 1716190681Snwhitehorn vm_paddr_t pa; 1717190681Snwhitehorn 1718205370Snwhitehorn /* 1719205370Snwhitehorn * Shortcut the direct-mapped case when applicable. We never put 1720205370Snwhitehorn * anything but 1:1 mappings below VM_MIN_KERNEL_ADDRESS. 1721205370Snwhitehorn */ 1722205370Snwhitehorn if (va < VM_MIN_KERNEL_ADDRESS) 1723205370Snwhitehorn return (va); 1724205370Snwhitehorn 1725190681Snwhitehorn PMAP_LOCK(kernel_pmap); 1726209975Snwhitehorn pvo = moea64_pvo_find_va(kernel_pmap, va); 1727209975Snwhitehorn KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR, 1728209975Snwhitehorn va)); 1729223471Sandreast pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va - PVO_VADDR(pvo)); 1730190681Snwhitehorn PMAP_UNLOCK(kernel_pmap); 1731190681Snwhitehorn return (pa); 1732190681Snwhitehorn} 1733190681Snwhitehorn 1734190681Snwhitehorn/* 1735190681Snwhitehorn * Remove a wired page from kernel virtual address space. 1736190681Snwhitehorn */ 1737190681Snwhitehornvoid 1738190681Snwhitehornmoea64_kremove(mmu_t mmu, vm_offset_t va) 1739190681Snwhitehorn{ 1740190681Snwhitehorn moea64_remove(mmu, kernel_pmap, va, va + PAGE_SIZE); 1741190681Snwhitehorn} 1742190681Snwhitehorn 1743190681Snwhitehorn/* 1744190681Snwhitehorn * Map a range of physical addresses into kernel virtual address space. 1745190681Snwhitehorn * 1746190681Snwhitehorn * The value passed in *virt is a suggested virtual address for the mapping. 1747190681Snwhitehorn * Architectures which can support a direct-mapped physical to virtual region 1748190681Snwhitehorn * can return the appropriate address within that region, leaving '*virt' 1749190681Snwhitehorn * unchanged. We cannot and therefore do not; *virt is updated with the 1750190681Snwhitehorn * first usable address after the mapped region. 1751190681Snwhitehorn */ 1752190681Snwhitehornvm_offset_t 1753236019Srajmoea64_map(mmu_t mmu, vm_offset_t *virt, vm_paddr_t pa_start, 1754236019Sraj vm_paddr_t pa_end, int prot) 1755190681Snwhitehorn{ 1756190681Snwhitehorn vm_offset_t sva, va; 1757190681Snwhitehorn 1758190681Snwhitehorn sva = *virt; 1759190681Snwhitehorn va = sva; 1760190681Snwhitehorn for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) 1761190681Snwhitehorn moea64_kenter(mmu, va, pa_start); 1762190681Snwhitehorn *virt = va; 1763190681Snwhitehorn 1764190681Snwhitehorn return (sva); 1765190681Snwhitehorn} 1766190681Snwhitehorn 1767190681Snwhitehorn/* 1768190681Snwhitehorn * Returns true if the pmap's pv is one of the first 1769190681Snwhitehorn * 16 pvs linked to from this page. This count may 1770190681Snwhitehorn * be changed upwards or downwards in the future; it 1771190681Snwhitehorn * is only necessary that true be returned for a small 1772190681Snwhitehorn * subset of pmaps for proper page aging. 1773190681Snwhitehorn */ 1774190681Snwhitehornboolean_t 1775190681Snwhitehornmoea64_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) 1776190681Snwhitehorn{ 1777190681Snwhitehorn int loops; 1778190681Snwhitehorn struct pvo_entry *pvo; 1779208990Salc boolean_t rv; 1780190681Snwhitehorn 1781224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1782208990Salc ("moea64_page_exists_quick: page %p is not managed", m)); 1783190681Snwhitehorn loops = 0; 1784208990Salc rv = FALSE; 1785233529Snwhitehorn LOCK_TABLE_RD(); 1786190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1787208990Salc if (pvo->pvo_pmap == pmap) { 1788208990Salc rv = TRUE; 1789208990Salc break; 1790208990Salc } 1791190681Snwhitehorn if (++loops >= 16) 1792190681Snwhitehorn break; 1793190681Snwhitehorn } 1794233529Snwhitehorn UNLOCK_TABLE_RD(); 1795208990Salc return (rv); 1796190681Snwhitehorn} 1797190681Snwhitehorn 1798190681Snwhitehorn/* 1799190681Snwhitehorn * Return the number of managed mappings to the given physical page 1800190681Snwhitehorn * that are wired. 1801190681Snwhitehorn */ 1802190681Snwhitehornint 1803190681Snwhitehornmoea64_page_wired_mappings(mmu_t mmu, vm_page_t m) 1804190681Snwhitehorn{ 1805190681Snwhitehorn struct pvo_entry *pvo; 1806190681Snwhitehorn int count; 1807190681Snwhitehorn 1808190681Snwhitehorn count = 0; 1809224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) 1810190681Snwhitehorn return (count); 1811233529Snwhitehorn LOCK_TABLE_RD(); 1812190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) 1813190681Snwhitehorn if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 1814190681Snwhitehorn count++; 1815233529Snwhitehorn UNLOCK_TABLE_RD(); 1816190681Snwhitehorn return (count); 1817190681Snwhitehorn} 1818190681Snwhitehorn 1819209975Snwhitehornstatic uintptr_t moea64_vsidcontext; 1820190681Snwhitehorn 1821209975Snwhitehornuintptr_t 1822209975Snwhitehornmoea64_get_unique_vsid(void) { 1823209975Snwhitehorn u_int entropy; 1824209975Snwhitehorn register_t hash; 1825209975Snwhitehorn uint32_t mask; 1826209975Snwhitehorn int i; 1827190681Snwhitehorn 1828190681Snwhitehorn entropy = 0; 1829190681Snwhitehorn __asm __volatile("mftb %0" : "=r"(entropy)); 1830190681Snwhitehorn 1831211967Snwhitehorn mtx_lock(&moea64_slb_mutex); 1832209975Snwhitehorn for (i = 0; i < NVSIDS; i += VSID_NBPW) { 1833209975Snwhitehorn u_int n; 1834190681Snwhitehorn 1835190681Snwhitehorn /* 1836190681Snwhitehorn * Create a new value by mutiplying by a prime and adding in 1837190681Snwhitehorn * entropy from the timebase register. This is to make the 1838190681Snwhitehorn * VSID more random so that the PT hash function collides 1839190681Snwhitehorn * less often. (Note that the prime casues gcc to do shifts 1840190681Snwhitehorn * instead of a multiply.) 1841190681Snwhitehorn */ 1842190681Snwhitehorn moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy; 1843209975Snwhitehorn hash = moea64_vsidcontext & (NVSIDS - 1); 1844190681Snwhitehorn if (hash == 0) /* 0 is special, avoid it */ 1845190681Snwhitehorn continue; 1846190681Snwhitehorn n = hash >> 5; 1847190681Snwhitehorn mask = 1 << (hash & (VSID_NBPW - 1)); 1848209975Snwhitehorn hash = (moea64_vsidcontext & VSID_HASHMASK); 1849190681Snwhitehorn if (moea64_vsid_bitmap[n] & mask) { /* collision? */ 1850190681Snwhitehorn /* anything free in this bucket? */ 1851190681Snwhitehorn if (moea64_vsid_bitmap[n] == 0xffffffff) { 1852190681Snwhitehorn entropy = (moea64_vsidcontext >> 20); 1853190681Snwhitehorn continue; 1854190681Snwhitehorn } 1855212322Snwhitehorn i = ffs(~moea64_vsid_bitmap[n]) - 1; 1856190681Snwhitehorn mask = 1 << i; 1857209975Snwhitehorn hash &= VSID_HASHMASK & ~(VSID_NBPW - 1); 1858190681Snwhitehorn hash |= i; 1859190681Snwhitehorn } 1860212322Snwhitehorn KASSERT(!(moea64_vsid_bitmap[n] & mask), 1861212331Snwhitehorn ("Allocating in-use VSID %#zx\n", hash)); 1862190681Snwhitehorn moea64_vsid_bitmap[n] |= mask; 1863211967Snwhitehorn mtx_unlock(&moea64_slb_mutex); 1864209975Snwhitehorn return (hash); 1865190681Snwhitehorn } 1866190681Snwhitehorn 1867211967Snwhitehorn mtx_unlock(&moea64_slb_mutex); 1868209975Snwhitehorn panic("%s: out of segments",__func__); 1869190681Snwhitehorn} 1870190681Snwhitehorn 1871209975Snwhitehorn#ifdef __powerpc64__ 1872209975Snwhitehornvoid 1873209975Snwhitehornmoea64_pinit(mmu_t mmu, pmap_t pmap) 1874209975Snwhitehorn{ 1875254667Skib 1876235689Snwhitehorn RB_INIT(&pmap->pmap_pvo); 1877209975Snwhitehorn 1878212715Snwhitehorn pmap->pm_slb_tree_root = slb_alloc_tree(); 1879209975Snwhitehorn pmap->pm_slb = slb_alloc_user_cache(); 1880212722Snwhitehorn pmap->pm_slb_len = 0; 1881209975Snwhitehorn} 1882209975Snwhitehorn#else 1883209975Snwhitehornvoid 1884209975Snwhitehornmoea64_pinit(mmu_t mmu, pmap_t pmap) 1885209975Snwhitehorn{ 1886209975Snwhitehorn int i; 1887212308Snwhitehorn uint32_t hash; 1888209975Snwhitehorn 1889235689Snwhitehorn RB_INIT(&pmap->pmap_pvo); 1890209975Snwhitehorn 1891209975Snwhitehorn if (pmap_bootstrapped) 1892209975Snwhitehorn pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, 1893209975Snwhitehorn (vm_offset_t)pmap); 1894209975Snwhitehorn else 1895209975Snwhitehorn pmap->pmap_phys = pmap; 1896209975Snwhitehorn 1897209975Snwhitehorn /* 1898209975Snwhitehorn * Allocate some segment registers for this pmap. 1899209975Snwhitehorn */ 1900209975Snwhitehorn hash = moea64_get_unique_vsid(); 1901209975Snwhitehorn 1902209975Snwhitehorn for (i = 0; i < 16; i++) 1903209975Snwhitehorn pmap->pm_sr[i] = VSID_MAKE(i, hash); 1904212308Snwhitehorn 1905212308Snwhitehorn KASSERT(pmap->pm_sr[0] != 0, ("moea64_pinit: pm_sr[0] = 0")); 1906209975Snwhitehorn} 1907209975Snwhitehorn#endif 1908209975Snwhitehorn 1909190681Snwhitehorn/* 1910190681Snwhitehorn * Initialize the pmap associated with process 0. 1911190681Snwhitehorn */ 1912190681Snwhitehornvoid 1913190681Snwhitehornmoea64_pinit0(mmu_t mmu, pmap_t pm) 1914190681Snwhitehorn{ 1915254667Skib 1916254667Skib PMAP_LOCK_INIT(pm); 1917190681Snwhitehorn moea64_pinit(mmu, pm); 1918190681Snwhitehorn bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1919190681Snwhitehorn} 1920190681Snwhitehorn 1921190681Snwhitehorn/* 1922190681Snwhitehorn * Set the physical protection on the specified range of this map as requested. 1923190681Snwhitehorn */ 1924233011Snwhitehornstatic void 1925233011Snwhitehornmoea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot) 1926233011Snwhitehorn{ 1927233011Snwhitehorn uintptr_t pt; 1928233949Snwhitehorn struct vm_page *pg; 1929233436Snwhitehorn uint64_t oldlo; 1930233011Snwhitehorn 1931233529Snwhitehorn PMAP_LOCK_ASSERT(pm, MA_OWNED); 1932233529Snwhitehorn 1933233011Snwhitehorn /* 1934233011Snwhitehorn * Grab the PTE pointer before we diddle with the cached PTE 1935233011Snwhitehorn * copy. 1936233011Snwhitehorn */ 1937233011Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 1938233011Snwhitehorn 1939233011Snwhitehorn /* 1940233011Snwhitehorn * Change the protection of the page. 1941233011Snwhitehorn */ 1942233436Snwhitehorn oldlo = pvo->pvo_pte.lpte.pte_lo; 1943233011Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP; 1944233011Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC; 1945233011Snwhitehorn if ((prot & VM_PROT_EXECUTE) == 0) 1946233011Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC; 1947233436Snwhitehorn if (prot & VM_PROT_WRITE) 1948233436Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= LPTE_BW; 1949233436Snwhitehorn else 1950233436Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= LPTE_BR; 1951233011Snwhitehorn 1952233949Snwhitehorn pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); 1953233949Snwhitehorn 1954233011Snwhitehorn /* 1955233011Snwhitehorn * If the PVO is in the page table, update that pte as well. 1956233011Snwhitehorn */ 1957234155Snwhitehorn if (pt != -1) 1958233011Snwhitehorn MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte, 1959233011Snwhitehorn pvo->pvo_vpn); 1960234155Snwhitehorn if (pm != kernel_pmap && pg != NULL && !(pg->aflags & PGA_EXECUTABLE) && 1961234155Snwhitehorn (pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { 1962234155Snwhitehorn if ((pg->oflags & VPO_UNMANAGED) == 0) 1963233949Snwhitehorn vm_page_aflag_set(pg, PGA_EXECUTABLE); 1964234155Snwhitehorn moea64_syncicache(mmu, pm, PVO_VADDR(pvo), 1965234155Snwhitehorn pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE); 1966233011Snwhitehorn } 1967233434Snwhitehorn 1968233434Snwhitehorn /* 1969233436Snwhitehorn * Update vm about the REF/CHG bits if the page is managed and we have 1970233436Snwhitehorn * removed write access. 1971233434Snwhitehorn */ 1972233436Snwhitehorn if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && 1973253272Snwhitehorn (oldlo & LPTE_PP) != LPTE_BR && !(prot & VM_PROT_WRITE)) { 1974233434Snwhitehorn if (pg != NULL) { 1975233434Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG) 1976233434Snwhitehorn vm_page_dirty(pg); 1977233434Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF) 1978233434Snwhitehorn vm_page_aflag_set(pg, PGA_REFERENCED); 1979233434Snwhitehorn } 1980233434Snwhitehorn } 1981233011Snwhitehorn} 1982233011Snwhitehorn 1983190681Snwhitehornvoid 1984190681Snwhitehornmoea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, 1985190681Snwhitehorn vm_prot_t prot) 1986190681Snwhitehorn{ 1987235689Snwhitehorn struct pvo_entry *pvo, *tpvo, key; 1988190681Snwhitehorn 1989233011Snwhitehorn CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, 1990233011Snwhitehorn sva, eva, prot); 1991190681Snwhitehorn 1992190681Snwhitehorn KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1993190681Snwhitehorn ("moea64_protect: non current pmap")); 1994190681Snwhitehorn 1995190681Snwhitehorn if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1996190681Snwhitehorn moea64_remove(mmu, pm, sva, eva); 1997190681Snwhitehorn return; 1998190681Snwhitehorn } 1999190681Snwhitehorn 2000233529Snwhitehorn LOCK_TABLE_RD(); 2001190681Snwhitehorn PMAP_LOCK(pm); 2002235689Snwhitehorn key.pvo_vaddr = sva; 2003235689Snwhitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 2004235689Snwhitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 2005235689Snwhitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 2006235689Snwhitehorn moea64_pvo_protect(mmu, pm, pvo, prot); 2007190681Snwhitehorn } 2008233529Snwhitehorn UNLOCK_TABLE_RD(); 2009190681Snwhitehorn PMAP_UNLOCK(pm); 2010190681Snwhitehorn} 2011190681Snwhitehorn 2012190681Snwhitehorn/* 2013190681Snwhitehorn * Map a list of wired pages into kernel virtual address space. This is 2014190681Snwhitehorn * intended for temporary mappings which do not need page modification or 2015190681Snwhitehorn * references recorded. Existing mappings in the region are overwritten. 2016190681Snwhitehorn */ 2017190681Snwhitehornvoid 2018190681Snwhitehornmoea64_qenter(mmu_t mmu, vm_offset_t va, vm_page_t *m, int count) 2019190681Snwhitehorn{ 2020190681Snwhitehorn while (count-- > 0) { 2021190681Snwhitehorn moea64_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); 2022190681Snwhitehorn va += PAGE_SIZE; 2023190681Snwhitehorn m++; 2024190681Snwhitehorn } 2025190681Snwhitehorn} 2026190681Snwhitehorn 2027190681Snwhitehorn/* 2028190681Snwhitehorn * Remove page mappings from kernel virtual address space. Intended for 2029190681Snwhitehorn * temporary mappings entered by moea64_qenter. 2030190681Snwhitehorn */ 2031190681Snwhitehornvoid 2032190681Snwhitehornmoea64_qremove(mmu_t mmu, vm_offset_t va, int count) 2033190681Snwhitehorn{ 2034190681Snwhitehorn while (count-- > 0) { 2035190681Snwhitehorn moea64_kremove(mmu, va); 2036190681Snwhitehorn va += PAGE_SIZE; 2037190681Snwhitehorn } 2038190681Snwhitehorn} 2039190681Snwhitehorn 2040190681Snwhitehornvoid 2041209975Snwhitehornmoea64_release_vsid(uint64_t vsid) 2042209975Snwhitehorn{ 2043212044Snwhitehorn int idx, mask; 2044209975Snwhitehorn 2045212044Snwhitehorn mtx_lock(&moea64_slb_mutex); 2046212044Snwhitehorn idx = vsid & (NVSIDS-1); 2047212044Snwhitehorn mask = 1 << (idx % VSID_NBPW); 2048212044Snwhitehorn idx /= VSID_NBPW; 2049212308Snwhitehorn KASSERT(moea64_vsid_bitmap[idx] & mask, 2050212308Snwhitehorn ("Freeing unallocated VSID %#jx", vsid)); 2051212044Snwhitehorn moea64_vsid_bitmap[idx] &= ~mask; 2052212044Snwhitehorn mtx_unlock(&moea64_slb_mutex); 2053209975Snwhitehorn} 2054209975Snwhitehorn 2055209975Snwhitehorn 2056209975Snwhitehornvoid 2057190681Snwhitehornmoea64_release(mmu_t mmu, pmap_t pmap) 2058190681Snwhitehorn{ 2059190681Snwhitehorn 2060190681Snwhitehorn /* 2061209975Snwhitehorn * Free segment registers' VSIDs 2062190681Snwhitehorn */ 2063209975Snwhitehorn #ifdef __powerpc64__ 2064212715Snwhitehorn slb_free_tree(pmap); 2065209975Snwhitehorn slb_free_user_cache(pmap->pm_slb); 2066209975Snwhitehorn #else 2067212308Snwhitehorn KASSERT(pmap->pm_sr[0] != 0, ("moea64_release: pm_sr[0] = 0")); 2068190681Snwhitehorn 2069212308Snwhitehorn moea64_release_vsid(VSID_TO_HASH(pmap->pm_sr[0])); 2070209975Snwhitehorn #endif 2071190681Snwhitehorn} 2072190681Snwhitehorn 2073190681Snwhitehorn/* 2074233017Snwhitehorn * Remove all pages mapped by the specified pmap 2075233017Snwhitehorn */ 2076233017Snwhitehornvoid 2077233017Snwhitehornmoea64_remove_pages(mmu_t mmu, pmap_t pm) 2078233017Snwhitehorn{ 2079233017Snwhitehorn struct pvo_entry *pvo, *tpvo; 2080233017Snwhitehorn 2081233529Snwhitehorn LOCK_TABLE_WR(); 2082233017Snwhitehorn PMAP_LOCK(pm); 2083235689Snwhitehorn RB_FOREACH_SAFE(pvo, pvo_tree, &pm->pmap_pvo, tpvo) { 2084233434Snwhitehorn if (!(pvo->pvo_vaddr & PVO_WIRED)) 2085233434Snwhitehorn moea64_pvo_remove(mmu, pvo); 2086233434Snwhitehorn } 2087233529Snwhitehorn UNLOCK_TABLE_WR(); 2088233017Snwhitehorn PMAP_UNLOCK(pm); 2089233017Snwhitehorn} 2090233017Snwhitehorn 2091233017Snwhitehorn/* 2092190681Snwhitehorn * Remove the given range of addresses from the specified map. 2093190681Snwhitehorn */ 2094190681Snwhitehornvoid 2095190681Snwhitehornmoea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) 2096190681Snwhitehorn{ 2097235689Snwhitehorn struct pvo_entry *pvo, *tpvo, key; 2098190681Snwhitehorn 2099233011Snwhitehorn /* 2100233011Snwhitehorn * Perform an unsynchronized read. This is, however, safe. 2101233011Snwhitehorn */ 2102233011Snwhitehorn if (pm->pm_stats.resident_count == 0) 2103233011Snwhitehorn return; 2104233011Snwhitehorn 2105233529Snwhitehorn LOCK_TABLE_WR(); 2106190681Snwhitehorn PMAP_LOCK(pm); 2107235689Snwhitehorn key.pvo_vaddr = sva; 2108235689Snwhitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 2109235689Snwhitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 2110235689Snwhitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 2111235689Snwhitehorn moea64_pvo_remove(mmu, pvo); 2112190681Snwhitehorn } 2113233529Snwhitehorn UNLOCK_TABLE_WR(); 2114190681Snwhitehorn PMAP_UNLOCK(pm); 2115190681Snwhitehorn} 2116190681Snwhitehorn 2117190681Snwhitehorn/* 2118190681Snwhitehorn * Remove physical page from all pmaps in which it resides. moea64_pvo_remove() 2119190681Snwhitehorn * will reflect changes in pte's back to the vm_page. 2120190681Snwhitehorn */ 2121190681Snwhitehornvoid 2122190681Snwhitehornmoea64_remove_all(mmu_t mmu, vm_page_t m) 2123190681Snwhitehorn{ 2124190681Snwhitehorn struct pvo_entry *pvo, *next_pvo; 2125190681Snwhitehorn pmap_t pmap; 2126190681Snwhitehorn 2127233529Snwhitehorn LOCK_TABLE_WR(); 2128233949Snwhitehorn LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) { 2129190681Snwhitehorn pmap = pvo->pvo_pmap; 2130190681Snwhitehorn PMAP_LOCK(pmap); 2131216174Snwhitehorn moea64_pvo_remove(mmu, pvo); 2132190681Snwhitehorn PMAP_UNLOCK(pmap); 2133190681Snwhitehorn } 2134233529Snwhitehorn UNLOCK_TABLE_WR(); 2135233434Snwhitehorn if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m)) 2136204042Snwhitehorn vm_page_dirty(m); 2137225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 2138233949Snwhitehorn vm_page_aflag_clear(m, PGA_EXECUTABLE); 2139190681Snwhitehorn} 2140190681Snwhitehorn 2141190681Snwhitehorn/* 2142190681Snwhitehorn * Allocate a physical page of memory directly from the phys_avail map. 2143190681Snwhitehorn * Can only be called from moea64_bootstrap before avail start and end are 2144190681Snwhitehorn * calculated. 2145190681Snwhitehorn */ 2146216174Snwhitehornvm_offset_t 2147190681Snwhitehornmoea64_bootstrap_alloc(vm_size_t size, u_int align) 2148190681Snwhitehorn{ 2149190681Snwhitehorn vm_offset_t s, e; 2150190681Snwhitehorn int i, j; 2151190681Snwhitehorn 2152190681Snwhitehorn size = round_page(size); 2153190681Snwhitehorn for (i = 0; phys_avail[i + 1] != 0; i += 2) { 2154190681Snwhitehorn if (align != 0) 2155190681Snwhitehorn s = (phys_avail[i] + align - 1) & ~(align - 1); 2156190681Snwhitehorn else 2157190681Snwhitehorn s = phys_avail[i]; 2158190681Snwhitehorn e = s + size; 2159190681Snwhitehorn 2160190681Snwhitehorn if (s < phys_avail[i] || e > phys_avail[i + 1]) 2161190681Snwhitehorn continue; 2162190681Snwhitehorn 2163215159Snwhitehorn if (s + size > platform_real_maxaddr()) 2164215159Snwhitehorn continue; 2165215159Snwhitehorn 2166190681Snwhitehorn if (s == phys_avail[i]) { 2167190681Snwhitehorn phys_avail[i] += size; 2168190681Snwhitehorn } else if (e == phys_avail[i + 1]) { 2169190681Snwhitehorn phys_avail[i + 1] -= size; 2170190681Snwhitehorn } else { 2171190681Snwhitehorn for (j = phys_avail_count * 2; j > i; j -= 2) { 2172190681Snwhitehorn phys_avail[j] = phys_avail[j - 2]; 2173190681Snwhitehorn phys_avail[j + 1] = phys_avail[j - 1]; 2174190681Snwhitehorn } 2175190681Snwhitehorn 2176190681Snwhitehorn phys_avail[i + 3] = phys_avail[i + 1]; 2177190681Snwhitehorn phys_avail[i + 1] = s; 2178190681Snwhitehorn phys_avail[i + 2] = e; 2179190681Snwhitehorn phys_avail_count++; 2180190681Snwhitehorn } 2181190681Snwhitehorn 2182190681Snwhitehorn return (s); 2183190681Snwhitehorn } 2184190681Snwhitehorn panic("moea64_bootstrap_alloc: could not allocate memory"); 2185190681Snwhitehorn} 2186190681Snwhitehorn 2187190681Snwhitehornstatic int 2188216174Snwhitehornmoea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone, 2189216174Snwhitehorn struct pvo_head *pvo_head, vm_offset_t va, vm_offset_t pa, 2190270439Skib uint64_t pte_lo, int flags, int8_t psind __unused) 2191190681Snwhitehorn{ 2192190681Snwhitehorn struct pvo_entry *pvo; 2193270920Skib uintptr_t pt; 2194190681Snwhitehorn uint64_t vsid; 2195190681Snwhitehorn int first; 2196190681Snwhitehorn u_int ptegidx; 2197190681Snwhitehorn int i; 2198190681Snwhitehorn int bootstrap; 2199190681Snwhitehorn 2200190681Snwhitehorn /* 2201190681Snwhitehorn * One nasty thing that can happen here is that the UMA calls to 2202190681Snwhitehorn * allocate new PVOs need to map more memory, which calls pvo_enter(), 2203190681Snwhitehorn * which calls UMA... 2204190681Snwhitehorn * 2205190681Snwhitehorn * We break the loop by detecting recursion and allocating out of 2206190681Snwhitehorn * the bootstrap pool. 2207190681Snwhitehorn */ 2208190681Snwhitehorn 2209190681Snwhitehorn first = 0; 2210190681Snwhitehorn bootstrap = (flags & PVO_BOOTSTRAP); 2211190681Snwhitehorn 2212190681Snwhitehorn if (!moea64_initialized) 2213190681Snwhitehorn bootstrap = 1; 2214190681Snwhitehorn 2215233529Snwhitehorn PMAP_LOCK_ASSERT(pm, MA_OWNED); 2216233529Snwhitehorn rw_assert(&moea64_table_lock, RA_WLOCKED); 2217233529Snwhitehorn 2218190681Snwhitehorn /* 2219190681Snwhitehorn * Compute the PTE Group index. 2220190681Snwhitehorn */ 2221190681Snwhitehorn va &= ~ADDR_POFF; 2222190681Snwhitehorn vsid = va_to_vsid(pm, va); 2223209975Snwhitehorn ptegidx = va_to_pteg(vsid, va, flags & PVO_LARGE); 2224190681Snwhitehorn 2225190681Snwhitehorn /* 2226190681Snwhitehorn * Remove any existing mapping for this page. Reuse the pvo entry if 2227190681Snwhitehorn * there is a mapping. 2228190681Snwhitehorn */ 2229212363Snwhitehorn moea64_pvo_enter_calls++; 2230212363Snwhitehorn 2231190681Snwhitehorn LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { 2232190681Snwhitehorn if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 2233190681Snwhitehorn if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa && 2234217341Snwhitehorn (pvo->pvo_pte.lpte.pte_lo & (LPTE_NOEXEC | LPTE_PP)) 2235217341Snwhitehorn == (pte_lo & (LPTE_NOEXEC | LPTE_PP))) { 2236270920Skib /* 2237270920Skib * The physical page and protection are not 2238270920Skib * changing. Instead, this may be a request 2239270920Skib * to change the mapping's wired attribute. 2240270920Skib */ 2241270920Skib pt = -1; 2242270920Skib if ((flags & PVO_WIRED) != 0 && 2243270920Skib (pvo->pvo_vaddr & PVO_WIRED) == 0) { 2244270920Skib pt = MOEA64_PVO_TO_PTE(mmu, pvo); 2245270920Skib pvo->pvo_vaddr |= PVO_WIRED; 2246270920Skib pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED; 2247270920Skib pm->pm_stats.wired_count++; 2248270920Skib } else if ((flags & PVO_WIRED) == 0 && 2249270920Skib (pvo->pvo_vaddr & PVO_WIRED) != 0) { 2250270920Skib pt = MOEA64_PVO_TO_PTE(mmu, pvo); 2251270920Skib pvo->pvo_vaddr &= ~PVO_WIRED; 2252270920Skib pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED; 2253270920Skib pm->pm_stats.wired_count--; 2254270920Skib } 2255209975Snwhitehorn if (!(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) { 2256270920Skib KASSERT(pt == -1, 2257270920Skib ("moea64_pvo_enter: valid pt")); 2258209975Snwhitehorn /* Re-insert if spilled */ 2259216174Snwhitehorn i = MOEA64_PTE_INSERT(mmu, ptegidx, 2260209975Snwhitehorn &pvo->pvo_pte.lpte); 2261209975Snwhitehorn if (i >= 0) 2262209975Snwhitehorn PVO_PTEGIDX_SET(pvo, i); 2263209975Snwhitehorn moea64_pte_overflow--; 2264270920Skib } else if (pt != -1) { 2265270920Skib /* 2266270920Skib * The PTE's wired attribute is not a 2267270920Skib * hardware feature, so there is no 2268270920Skib * need to invalidate any TLB entries. 2269270920Skib */ 2270270920Skib MOEA64_PTE_CHANGE(mmu, pt, 2271270920Skib &pvo->pvo_pte.lpte, pvo->pvo_vpn); 2272209975Snwhitehorn } 2273190681Snwhitehorn return (0); 2274190681Snwhitehorn } 2275216174Snwhitehorn moea64_pvo_remove(mmu, pvo); 2276190681Snwhitehorn break; 2277190681Snwhitehorn } 2278190681Snwhitehorn } 2279190681Snwhitehorn 2280190681Snwhitehorn /* 2281190681Snwhitehorn * If we aren't overwriting a mapping, try to allocate. 2282190681Snwhitehorn */ 2283190681Snwhitehorn if (bootstrap) { 2284190681Snwhitehorn if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) { 2285209975Snwhitehorn panic("moea64_enter: bpvo pool exhausted, %d, %d, %zd", 2286190681Snwhitehorn moea64_bpvo_pool_index, BPVO_POOL_SIZE, 2287190681Snwhitehorn BPVO_POOL_SIZE * sizeof(struct pvo_entry)); 2288190681Snwhitehorn } 2289190681Snwhitehorn pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index]; 2290190681Snwhitehorn moea64_bpvo_pool_index++; 2291190681Snwhitehorn bootstrap = 1; 2292190681Snwhitehorn } else { 2293190681Snwhitehorn pvo = uma_zalloc(zone, M_NOWAIT); 2294190681Snwhitehorn } 2295190681Snwhitehorn 2296233529Snwhitehorn if (pvo == NULL) 2297190681Snwhitehorn return (ENOMEM); 2298190681Snwhitehorn 2299190681Snwhitehorn moea64_pvo_entries++; 2300190681Snwhitehorn pvo->pvo_vaddr = va; 2301209975Snwhitehorn pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT) 2302209975Snwhitehorn | (vsid << 16); 2303190681Snwhitehorn pvo->pvo_pmap = pm; 2304190681Snwhitehorn LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink); 2305190681Snwhitehorn pvo->pvo_vaddr &= ~ADDR_POFF; 2306190681Snwhitehorn 2307190681Snwhitehorn if (flags & PVO_WIRED) 2308190681Snwhitehorn pvo->pvo_vaddr |= PVO_WIRED; 2309235689Snwhitehorn if (pvo_head != NULL) 2310190681Snwhitehorn pvo->pvo_vaddr |= PVO_MANAGED; 2311190681Snwhitehorn if (bootstrap) 2312190681Snwhitehorn pvo->pvo_vaddr |= PVO_BOOTSTRAP; 2313209975Snwhitehorn if (flags & PVO_LARGE) 2314209975Snwhitehorn pvo->pvo_vaddr |= PVO_LARGE; 2315190681Snwhitehorn 2316190681Snwhitehorn moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va, 2317209975Snwhitehorn (uint64_t)(pa) | pte_lo, flags); 2318190681Snwhitehorn 2319190681Snwhitehorn /* 2320228412Snwhitehorn * Add to pmap list 2321228412Snwhitehorn */ 2322235689Snwhitehorn RB_INSERT(pvo_tree, &pm->pmap_pvo, pvo); 2323228412Snwhitehorn 2324228412Snwhitehorn /* 2325190681Snwhitehorn * Remember if the list was empty and therefore will be the first 2326190681Snwhitehorn * item. 2327190681Snwhitehorn */ 2328235689Snwhitehorn if (pvo_head != NULL) { 2329235689Snwhitehorn if (LIST_FIRST(pvo_head) == NULL) 2330235689Snwhitehorn first = 1; 2331235689Snwhitehorn LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 2332235689Snwhitehorn } 2333190681Snwhitehorn 2334209975Snwhitehorn if (pvo->pvo_vaddr & PVO_WIRED) { 2335209975Snwhitehorn pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED; 2336190681Snwhitehorn pm->pm_stats.wired_count++; 2337209975Snwhitehorn } 2338190681Snwhitehorn pm->pm_stats.resident_count++; 2339190681Snwhitehorn 2340190681Snwhitehorn /* 2341190681Snwhitehorn * We hope this succeeds but it isn't required. 2342190681Snwhitehorn */ 2343216174Snwhitehorn i = MOEA64_PTE_INSERT(mmu, ptegidx, &pvo->pvo_pte.lpte); 2344190681Snwhitehorn if (i >= 0) { 2345190681Snwhitehorn PVO_PTEGIDX_SET(pvo, i); 2346190681Snwhitehorn } else { 2347190681Snwhitehorn panic("moea64_pvo_enter: overflow"); 2348190681Snwhitehorn moea64_pte_overflow++; 2349190681Snwhitehorn } 2350190681Snwhitehorn 2351204042Snwhitehorn if (pm == kernel_pmap) 2352204042Snwhitehorn isync(); 2353204042Snwhitehorn 2354209975Snwhitehorn#ifdef __powerpc64__ 2355209975Snwhitehorn /* 2356209975Snwhitehorn * Make sure all our bootstrap mappings are in the SLB as soon 2357209975Snwhitehorn * as virtual memory is switched on. 2358209975Snwhitehorn */ 2359209975Snwhitehorn if (!pmap_bootstrapped) 2360209975Snwhitehorn moea64_bootstrap_slb_prefault(va, flags & PVO_LARGE); 2361209975Snwhitehorn#endif 2362209975Snwhitehorn 2363190681Snwhitehorn return (first ? ENOENT : 0); 2364190681Snwhitehorn} 2365190681Snwhitehorn 2366190681Snwhitehornstatic void 2367216174Snwhitehornmoea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo) 2368190681Snwhitehorn{ 2369233949Snwhitehorn struct vm_page *pg; 2370216174Snwhitehorn uintptr_t pt; 2371190681Snwhitehorn 2372233529Snwhitehorn PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 2373233529Snwhitehorn rw_assert(&moea64_table_lock, RA_WLOCKED); 2374233529Snwhitehorn 2375190681Snwhitehorn /* 2376190681Snwhitehorn * If there is an active pte entry, we need to deactivate it (and 2377190681Snwhitehorn * save the ref & cfg bits). 2378190681Snwhitehorn */ 2379216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 2380216174Snwhitehorn if (pt != -1) { 2381216174Snwhitehorn MOEA64_PTE_UNSET(mmu, pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); 2382190681Snwhitehorn PVO_PTEGIDX_CLR(pvo); 2383190681Snwhitehorn } else { 2384190681Snwhitehorn moea64_pte_overflow--; 2385190681Snwhitehorn } 2386190681Snwhitehorn 2387190681Snwhitehorn /* 2388190681Snwhitehorn * Update our statistics. 2389190681Snwhitehorn */ 2390190681Snwhitehorn pvo->pvo_pmap->pm_stats.resident_count--; 2391204042Snwhitehorn if (pvo->pvo_vaddr & PVO_WIRED) 2392190681Snwhitehorn pvo->pvo_pmap->pm_stats.wired_count--; 2393190681Snwhitehorn 2394190681Snwhitehorn /* 2395235689Snwhitehorn * Remove this PVO from the pmap list. 2396233529Snwhitehorn */ 2397235689Snwhitehorn RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo); 2398233529Snwhitehorn 2399233529Snwhitehorn /* 2400233529Snwhitehorn * Remove this from the overflow list and return it to the pool 2401233529Snwhitehorn * if we aren't going to reuse it. 2402233529Snwhitehorn */ 2403233529Snwhitehorn LIST_REMOVE(pvo, pvo_olink); 2404233529Snwhitehorn 2405233529Snwhitehorn /* 2406233434Snwhitehorn * Update vm about the REF/CHG bits if the page is managed. 2407190681Snwhitehorn */ 2408233949Snwhitehorn pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); 2409233949Snwhitehorn 2410234155Snwhitehorn if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && pg != NULL) { 2411235689Snwhitehorn LIST_REMOVE(pvo, pvo_vlink); 2412234155Snwhitehorn if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { 2413233434Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG) 2414233434Snwhitehorn vm_page_dirty(pg); 2415233434Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF) 2416233434Snwhitehorn vm_page_aflag_set(pg, PGA_REFERENCED); 2417233529Snwhitehorn if (LIST_EMPTY(vm_page_to_pvoh(pg))) 2418233529Snwhitehorn vm_page_aflag_clear(pg, PGA_WRITEABLE); 2419190681Snwhitehorn } 2420234155Snwhitehorn if (LIST_EMPTY(vm_page_to_pvoh(pg))) 2421234155Snwhitehorn vm_page_aflag_clear(pg, PGA_EXECUTABLE); 2422190681Snwhitehorn } 2423190681Snwhitehorn 2424212363Snwhitehorn moea64_pvo_entries--; 2425212363Snwhitehorn moea64_pvo_remove_calls++; 2426212363Snwhitehorn 2427190681Snwhitehorn if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) 2428204042Snwhitehorn uma_zfree((pvo->pvo_vaddr & PVO_MANAGED) ? moea64_mpvo_zone : 2429190681Snwhitehorn moea64_upvo_zone, pvo); 2430190681Snwhitehorn} 2431190681Snwhitehorn 2432190681Snwhitehornstatic struct pvo_entry * 2433209975Snwhitehornmoea64_pvo_find_va(pmap_t pm, vm_offset_t va) 2434190681Snwhitehorn{ 2435235689Snwhitehorn struct pvo_entry key; 2436190681Snwhitehorn 2437235689Snwhitehorn key.pvo_vaddr = va & ~ADDR_POFF; 2438235689Snwhitehorn return (RB_FIND(pvo_tree, &pm->pmap_pvo, &key)); 2439190681Snwhitehorn} 2440190681Snwhitehorn 2441190681Snwhitehornstatic boolean_t 2442216174Snwhitehornmoea64_query_bit(mmu_t mmu, vm_page_t m, u_int64_t ptebit) 2443190681Snwhitehorn{ 2444190681Snwhitehorn struct pvo_entry *pvo; 2445216174Snwhitehorn uintptr_t pt; 2446190681Snwhitehorn 2447233529Snwhitehorn LOCK_TABLE_RD(); 2448190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2449190681Snwhitehorn /* 2450233434Snwhitehorn * See if we saved the bit off. If so, return success. 2451190681Snwhitehorn */ 2452190681Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & ptebit) { 2453233529Snwhitehorn UNLOCK_TABLE_RD(); 2454190681Snwhitehorn return (TRUE); 2455190681Snwhitehorn } 2456190681Snwhitehorn } 2457190681Snwhitehorn 2458190681Snwhitehorn /* 2459190681Snwhitehorn * No luck, now go through the hard part of looking at the PTEs 2460190681Snwhitehorn * themselves. Sync so that any pending REF/CHG bits are flushed to 2461190681Snwhitehorn * the PTEs. 2462190681Snwhitehorn */ 2463216174Snwhitehorn powerpc_sync(); 2464190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2465190681Snwhitehorn 2466190681Snwhitehorn /* 2467190681Snwhitehorn * See if this pvo has a valid PTE. if so, fetch the 2468190681Snwhitehorn * REF/CHG bits from the valid PTE. If the appropriate 2469233434Snwhitehorn * ptebit is set, return success. 2470190681Snwhitehorn */ 2471233529Snwhitehorn PMAP_LOCK(pvo->pvo_pmap); 2472216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 2473216174Snwhitehorn if (pt != -1) { 2474216174Snwhitehorn MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte); 2475190681Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & ptebit) { 2476233529Snwhitehorn PMAP_UNLOCK(pvo->pvo_pmap); 2477233529Snwhitehorn UNLOCK_TABLE_RD(); 2478190681Snwhitehorn return (TRUE); 2479190681Snwhitehorn } 2480190681Snwhitehorn } 2481233529Snwhitehorn PMAP_UNLOCK(pvo->pvo_pmap); 2482190681Snwhitehorn } 2483190681Snwhitehorn 2484233529Snwhitehorn UNLOCK_TABLE_RD(); 2485190681Snwhitehorn return (FALSE); 2486190681Snwhitehorn} 2487190681Snwhitehorn 2488190681Snwhitehornstatic u_int 2489216174Snwhitehornmoea64_clear_bit(mmu_t mmu, vm_page_t m, u_int64_t ptebit) 2490190681Snwhitehorn{ 2491190681Snwhitehorn u_int count; 2492190681Snwhitehorn struct pvo_entry *pvo; 2493216174Snwhitehorn uintptr_t pt; 2494190681Snwhitehorn 2495190681Snwhitehorn /* 2496190681Snwhitehorn * Sync so that any pending REF/CHG bits are flushed to the PTEs (so 2497190681Snwhitehorn * we can reset the right ones). note that since the pvo entries and 2498190681Snwhitehorn * list heads are accessed via BAT0 and are never placed in the page 2499190681Snwhitehorn * table, we don't have to worry about further accesses setting the 2500190681Snwhitehorn * REF/CHG bits. 2501190681Snwhitehorn */ 2502216174Snwhitehorn powerpc_sync(); 2503190681Snwhitehorn 2504190681Snwhitehorn /* 2505190681Snwhitehorn * For each pvo entry, clear the pvo's ptebit. If this pvo has a 2506190681Snwhitehorn * valid pte clear the ptebit from the valid pte. 2507190681Snwhitehorn */ 2508190681Snwhitehorn count = 0; 2509233529Snwhitehorn LOCK_TABLE_RD(); 2510190681Snwhitehorn LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2511233529Snwhitehorn PMAP_LOCK(pvo->pvo_pmap); 2512216174Snwhitehorn pt = MOEA64_PVO_TO_PTE(mmu, pvo); 2513216174Snwhitehorn if (pt != -1) { 2514216174Snwhitehorn MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte); 2515190681Snwhitehorn if (pvo->pvo_pte.lpte.pte_lo & ptebit) { 2516190681Snwhitehorn count++; 2517216174Snwhitehorn MOEA64_PTE_CLEAR(mmu, pt, &pvo->pvo_pte.lpte, 2518216174Snwhitehorn pvo->pvo_vpn, ptebit); 2519190681Snwhitehorn } 2520190681Snwhitehorn } 2521190681Snwhitehorn pvo->pvo_pte.lpte.pte_lo &= ~ptebit; 2522233529Snwhitehorn PMAP_UNLOCK(pvo->pvo_pmap); 2523190681Snwhitehorn } 2524190681Snwhitehorn 2525233529Snwhitehorn UNLOCK_TABLE_RD(); 2526190681Snwhitehorn return (count); 2527190681Snwhitehorn} 2528190681Snwhitehorn 2529190681Snwhitehornboolean_t 2530236019Srajmoea64_dev_direct_mapped(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2531190681Snwhitehorn{ 2532235689Snwhitehorn struct pvo_entry *pvo, key; 2533204296Snwhitehorn vm_offset_t ppa; 2534204296Snwhitehorn int error = 0; 2535204296Snwhitehorn 2536204296Snwhitehorn PMAP_LOCK(kernel_pmap); 2537235689Snwhitehorn key.pvo_vaddr = ppa = pa & ~ADDR_POFF; 2538235689Snwhitehorn for (pvo = RB_FIND(pvo_tree, &kernel_pmap->pmap_pvo, &key); 2539235689Snwhitehorn ppa < pa + size; ppa += PAGE_SIZE, 2540235689Snwhitehorn pvo = RB_NEXT(pvo_tree, &kernel_pmap->pmap_pvo, pvo)) { 2541204296Snwhitehorn if (pvo == NULL || 2542204296Snwhitehorn (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) != ppa) { 2543204296Snwhitehorn error = EFAULT; 2544204296Snwhitehorn break; 2545204296Snwhitehorn } 2546204296Snwhitehorn } 2547204296Snwhitehorn PMAP_UNLOCK(kernel_pmap); 2548204296Snwhitehorn 2549204296Snwhitehorn return (error); 2550190681Snwhitehorn} 2551190681Snwhitehorn 2552190681Snwhitehorn/* 2553190681Snwhitehorn * Map a set of physical memory pages into the kernel virtual 2554190681Snwhitehorn * address space. Return a pointer to where it is mapped. This 2555190681Snwhitehorn * routine is intended to be used for mapping device memory, 2556190681Snwhitehorn * NOT real memory. 2557190681Snwhitehorn */ 2558190681Snwhitehornvoid * 2559213307Snwhitehornmoea64_mapdev_attr(mmu_t mmu, vm_offset_t pa, vm_size_t size, vm_memattr_t ma) 2560190681Snwhitehorn{ 2561190681Snwhitehorn vm_offset_t va, tmpva, ppa, offset; 2562190681Snwhitehorn 2563190681Snwhitehorn ppa = trunc_page(pa); 2564190681Snwhitehorn offset = pa & PAGE_MASK; 2565233618Snwhitehorn size = roundup2(offset + size, PAGE_SIZE); 2566190681Snwhitehorn 2567254025Sjeff va = kva_alloc(size); 2568190681Snwhitehorn 2569190681Snwhitehorn if (!va) 2570190681Snwhitehorn panic("moea64_mapdev: Couldn't alloc kernel virtual memory"); 2571190681Snwhitehorn 2572190681Snwhitehorn for (tmpva = va; size > 0;) { 2573213307Snwhitehorn moea64_kenter_attr(mmu, tmpva, ppa, ma); 2574190681Snwhitehorn size -= PAGE_SIZE; 2575190681Snwhitehorn tmpva += PAGE_SIZE; 2576190681Snwhitehorn ppa += PAGE_SIZE; 2577190681Snwhitehorn } 2578190681Snwhitehorn 2579190681Snwhitehorn return ((void *)(va + offset)); 2580190681Snwhitehorn} 2581190681Snwhitehorn 2582213307Snwhitehornvoid * 2583236019Srajmoea64_mapdev(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2584213307Snwhitehorn{ 2585213307Snwhitehorn 2586213307Snwhitehorn return moea64_mapdev_attr(mmu, pa, size, VM_MEMATTR_DEFAULT); 2587213307Snwhitehorn} 2588213307Snwhitehorn 2589190681Snwhitehornvoid 2590190681Snwhitehornmoea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) 2591190681Snwhitehorn{ 2592190681Snwhitehorn vm_offset_t base, offset; 2593190681Snwhitehorn 2594190681Snwhitehorn base = trunc_page(va); 2595190681Snwhitehorn offset = va & PAGE_MASK; 2596233618Snwhitehorn size = roundup2(offset + size, PAGE_SIZE); 2597190681Snwhitehorn 2598254025Sjeff kva_free(base, size); 2599190681Snwhitehorn} 2600190681Snwhitehorn 2601216174Snwhitehornvoid 2602198341Smarcelmoea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) 2603198341Smarcel{ 2604198341Smarcel struct pvo_entry *pvo; 2605198341Smarcel vm_offset_t lim; 2606198341Smarcel vm_paddr_t pa; 2607198341Smarcel vm_size_t len; 2608198341Smarcel 2609198341Smarcel PMAP_LOCK(pm); 2610198341Smarcel while (sz > 0) { 2611198341Smarcel lim = round_page(va); 2612198341Smarcel len = MIN(lim - va, sz); 2613209975Snwhitehorn pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF); 2614222666Snwhitehorn if (pvo != NULL && !(pvo->pvo_pte.lpte.pte_lo & LPTE_I)) { 2615222666Snwhitehorn pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | 2616198341Smarcel (va & ADDR_POFF); 2617216174Snwhitehorn moea64_syncicache(mmu, pm, va, pa, len); 2618198341Smarcel } 2619198341Smarcel va += len; 2620198341Smarcel sz -= len; 2621198341Smarcel } 2622198341Smarcel PMAP_UNLOCK(pm); 2623198341Smarcel} 2624260672Sjhibbits 2625260672Sjhibbitsvm_offset_t 2626260672Sjhibbitsmoea64_dumpsys_map(mmu_t mmu, struct pmap_md *md, vm_size_t ofs, 2627260672Sjhibbits vm_size_t *sz) 2628260672Sjhibbits{ 2629260672Sjhibbits if (md->md_vaddr == ~0UL) 2630260672Sjhibbits return (md->md_paddr + ofs); 2631260672Sjhibbits else 2632260672Sjhibbits return (md->md_vaddr + ofs); 2633260672Sjhibbits} 2634260672Sjhibbits 2635260672Sjhibbitsstruct pmap_md * 2636260672Sjhibbitsmoea64_scan_md(mmu_t mmu, struct pmap_md *prev) 2637260672Sjhibbits{ 2638260672Sjhibbits static struct pmap_md md; 2639260672Sjhibbits struct pvo_entry *pvo; 2640260672Sjhibbits vm_offset_t va; 2641260672Sjhibbits 2642260672Sjhibbits if (dumpsys_minidump) { 2643260672Sjhibbits md.md_paddr = ~0UL; /* Minidumps use virtual addresses. */ 2644260672Sjhibbits if (prev == NULL) { 2645260672Sjhibbits /* 1st: kernel .data and .bss. */ 2646260672Sjhibbits md.md_index = 1; 2647260672Sjhibbits md.md_vaddr = trunc_page((uintptr_t)_etext); 2648260672Sjhibbits md.md_size = round_page((uintptr_t)_end) - md.md_vaddr; 2649260672Sjhibbits return (&md); 2650260672Sjhibbits } 2651260672Sjhibbits switch (prev->md_index) { 2652260672Sjhibbits case 1: 2653260672Sjhibbits /* 2nd: msgbuf and tables (see pmap_bootstrap()). */ 2654260672Sjhibbits md.md_index = 2; 2655260672Sjhibbits md.md_vaddr = (vm_offset_t)msgbufp->msg_ptr; 2656260672Sjhibbits md.md_size = round_page(msgbufp->msg_size); 2657260672Sjhibbits break; 2658260672Sjhibbits case 2: 2659260672Sjhibbits /* 3rd: kernel VM. */ 2660260672Sjhibbits va = prev->md_vaddr + prev->md_size; 2661260672Sjhibbits /* Find start of next chunk (from va). */ 2662260672Sjhibbits while (va < virtual_end) { 2663260672Sjhibbits /* Don't dump the buffer cache. */ 2664260672Sjhibbits if (va >= kmi.buffer_sva && 2665260672Sjhibbits va < kmi.buffer_eva) { 2666260672Sjhibbits va = kmi.buffer_eva; 2667260672Sjhibbits continue; 2668260672Sjhibbits } 2669260672Sjhibbits pvo = moea64_pvo_find_va(kernel_pmap, 2670260672Sjhibbits va & ~ADDR_POFF); 2671260672Sjhibbits if (pvo != NULL && 2672260672Sjhibbits (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) 2673260672Sjhibbits break; 2674260672Sjhibbits va += PAGE_SIZE; 2675260672Sjhibbits } 2676260672Sjhibbits if (va < virtual_end) { 2677260672Sjhibbits md.md_vaddr = va; 2678260672Sjhibbits va += PAGE_SIZE; 2679260672Sjhibbits /* Find last page in chunk. */ 2680260672Sjhibbits while (va < virtual_end) { 2681260672Sjhibbits /* Don't run into the buffer cache. */ 2682260672Sjhibbits if (va == kmi.buffer_sva) 2683260672Sjhibbits break; 2684260672Sjhibbits pvo = moea64_pvo_find_va(kernel_pmap, 2685260672Sjhibbits va & ~ADDR_POFF); 2686260672Sjhibbits if (pvo == NULL || 2687260672Sjhibbits !(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) 2688260672Sjhibbits break; 2689260672Sjhibbits va += PAGE_SIZE; 2690260672Sjhibbits } 2691260672Sjhibbits md.md_size = va - md.md_vaddr; 2692260672Sjhibbits break; 2693260672Sjhibbits } 2694260672Sjhibbits md.md_index = 3; 2695260672Sjhibbits /* FALLTHROUGH */ 2696260672Sjhibbits default: 2697260672Sjhibbits return (NULL); 2698260672Sjhibbits } 2699260672Sjhibbits } else { /* minidumps */ 2700260672Sjhibbits if (prev == NULL) { 2701260672Sjhibbits /* first physical chunk. */ 2702260672Sjhibbits md.md_paddr = pregions[0].mr_start; 2703260672Sjhibbits md.md_size = pregions[0].mr_size; 2704260672Sjhibbits md.md_vaddr = ~0UL; 2705260672Sjhibbits md.md_index = 1; 2706260672Sjhibbits } else if (md.md_index < pregions_sz) { 2707260672Sjhibbits md.md_paddr = pregions[md.md_index].mr_start; 2708260672Sjhibbits md.md_size = pregions[md.md_index].mr_size; 2709260672Sjhibbits md.md_vaddr = ~0UL; 2710260672Sjhibbits md.md_index++; 2711260672Sjhibbits } else { 2712260672Sjhibbits /* There's no next physical chunk. */ 2713260672Sjhibbits return (NULL); 2714260672Sjhibbits } 2715260672Sjhibbits } 2716260672Sjhibbits 2717260672Sjhibbits return (&md); 2718260672Sjhibbits} 2719