180709Sjake/*- 280709Sjake * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 380709Sjake * 480709Sjake * Redistribution and use in source and binary forms, with or without 580709Sjake * modification, are permitted provided that the following conditions 680709Sjake * are met: 780709Sjake * 1. Redistributions of source code must retain the above copyright 880709Sjake * notice, this list of conditions and the following disclaimer. 980709Sjake * 2. Redistributions in binary form must reproduce the above copyright 1080709Sjake * notice, this list of conditions and the following disclaimer in the 1180709Sjake * documentation and/or other materials provided with the distribution. 1280709Sjake * 3. Berkeley Software Design Inc's name may not be used to endorse or 1380709Sjake * promote products derived from this software without specific prior 1480709Sjake * written permission. 1580709Sjake * 1680709Sjake * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1780709Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1880709Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1980709Sjake * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2080709Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2180709Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2280709Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2380709Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2480709Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2580709Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2680709Sjake * SUCH DAMAGE. 2780709Sjake * 2880709Sjake * from BSDI: pmap.c,v 1.28.2.15 2000/04/27 03:10:31 cp Exp 2980709Sjake */ 3080709Sjake 31216803Smarius#include <sys/cdefs.h> 32216803Smarius__FBSDID("$FreeBSD$"); 33216803Smarius 3480709Sjake#include "opt_ddb.h" 3591167Sjake#include "opt_pmap.h" 3680709Sjake 3780709Sjake#include <sys/param.h> 3880709Sjake#include <sys/queue.h> 3985258Sjake#include <sys/ktr.h> 4080709Sjake#include <sys/lock.h> 4180709Sjake#include <sys/mutex.h> 4280709Sjake#include <sys/proc.h> 43236214Salc#include <sys/rwlock.h> 4491783Sjake#include <sys/smp.h> 4591167Sjake#include <sys/sysctl.h> 4680709Sjake#include <sys/systm.h> 4780709Sjake 48216803Smarius#include <vm/vm.h> 4980709Sjake#include <vm/vm_param.h> 5080709Sjake#include <vm/vm_kern.h> 5180709Sjake#include <vm/vm_page.h> 5280709Sjake#include <vm/vm_map.h> 5380709Sjake#include <vm/vm_object.h> 5480709Sjake#include <vm/vm_extern.h> 5580709Sjake#include <vm/vm_pageout.h> 5680709Sjake 5780709Sjake#include <machine/cpufunc.h> 5880709Sjake#include <machine/frame.h> 5980709Sjake#include <machine/trap.h> 6080709Sjake#include <machine/pmap.h> 6191783Sjake#include <machine/smp.h> 6280709Sjake#include <machine/tlb.h> 6380709Sjake#include <machine/tsb.h> 6480709Sjake#include <machine/tte.h> 6580709Sjake 6689046SjakeCTASSERT((1 << TTE_SHIFT) == sizeof(struct tte)); 67102040SjakeCTASSERT(TSB_BUCKET_MASK < (1 << 12)); 6880709Sjake 69108700SjakePMAP_STATS_VAR(tsb_nrepl); 70108700SjakePMAP_STATS_VAR(tsb_nlookup_k); 71108700SjakePMAP_STATS_VAR(tsb_nlookup_u); 72108700SjakePMAP_STATS_VAR(tsb_nenter_k); 73112879SjakePMAP_STATS_VAR(tsb_nenter_k_oc); 74108700SjakePMAP_STATS_VAR(tsb_nenter_u); 75112879SjakePMAP_STATS_VAR(tsb_nenter_u_oc); 76108700SjakePMAP_STATS_VAR(tsb_nforeach); 7791167Sjake 7888649Sjakestruct tte *tsb_kernel; 79101653Sjakevm_size_t tsb_kernel_mask; 80101653Sjakevm_size_t tsb_kernel_size; 81113238Sjakevm_paddr_t tsb_kernel_phys; 82216803Smariusu_int tsb_kernel_ldd_phys; 8380709Sjake 8488649Sjakestruct tte * 8588649Sjaketsb_tte_lookup(pmap_t pm, vm_offset_t va) 8680709Sjake{ 8788649Sjake struct tte *bucket; 8888649Sjake struct tte *tp; 89102040Sjake u_long sz; 9080709Sjake u_int i; 9180709Sjake 9282903Sjake if (pm == kernel_pmap) { 93108700Sjake PMAP_STATS_INC(tsb_nlookup_k); 9488649Sjake tp = tsb_kvtotte(va); 9599935Sjake if (tte_match(tp, va)) 9688649Sjake return (tp); 9788649Sjake } else { 98141712Salc PMAP_LOCK_ASSERT(pm, MA_OWNED); 99108700Sjake PMAP_STATS_INC(tsb_nlookup_u); 100102040Sjake for (sz = TS_MIN; sz <= TS_MAX; sz++) { 101102040Sjake bucket = tsb_vtobucket(pm, sz, va); 102102040Sjake for (i = 0; i < TSB_BUCKET_SIZE; i++) { 103102040Sjake tp = &bucket[i]; 104102040Sjake if (tte_match(tp, va)) 105102040Sjake return (tp); 106102040Sjake } 10780709Sjake } 10880709Sjake } 10980709Sjake return (NULL); 11080709Sjake} 11180709Sjake 11288649Sjakestruct tte * 113102040Sjaketsb_tte_enter(pmap_t pm, vm_page_t m, vm_offset_t va, u_long sz, u_long data) 11480709Sjake{ 11588649Sjake struct tte *bucket; 11688649Sjake struct tte *rtp; 11788649Sjake struct tte *tp; 11888649Sjake vm_offset_t ova; 11980709Sjake int b0; 12080709Sjake int i; 12180709Sjake 122170249Salc if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { 123210334Sattilio CTR5(KTR_SPARE2, 124174933Salc "tsb_tte_enter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", 125112879Sjake va, VM_PAGE_TO_PHYS(m), m->object, 126112879Sjake m->object ? m->object->type : -1, 127112879Sjake m->pindex); 128112879Sjake if (pm == kernel_pmap) 129112879Sjake PMAP_STATS_INC(tsb_nenter_k_oc); 130112879Sjake else 131112879Sjake PMAP_STATS_INC(tsb_nenter_u_oc); 132112879Sjake } 133112879Sjake 134236214Salc rw_assert(&tte_list_global_lock, RA_WLOCKED); 135133451Salc PMAP_LOCK_ASSERT(pm, MA_OWNED); 13688649Sjake if (pm == kernel_pmap) { 137108700Sjake PMAP_STATS_INC(tsb_nenter_k); 13888649Sjake tp = tsb_kvtotte(va); 13997447Sjake KASSERT((tp->tte_data & TD_V) == 0, 14097447Sjake ("tsb_tte_enter: replacing valid kernel mapping")); 14197447Sjake goto enter; 14288649Sjake } 143108700Sjake PMAP_STATS_INC(tsb_nenter_u); 14488649Sjake 145102040Sjake bucket = tsb_vtobucket(pm, sz, va); 14688649Sjake 14788649Sjake tp = NULL; 14888649Sjake rtp = NULL; 14988649Sjake b0 = rd(tick) & (TSB_BUCKET_SIZE - 1); 15080709Sjake i = b0; 15180709Sjake do { 15288649Sjake if ((bucket[i].tte_data & TD_V) == 0) { 15388649Sjake tp = &bucket[i]; 15484191Sjake break; 15580709Sjake } 15688649Sjake if (tp == NULL) { 15788649Sjake if ((bucket[i].tte_data & TD_REF) == 0) 15888649Sjake tp = &bucket[i]; 15988649Sjake else if (rtp == NULL) 16088649Sjake rtp = &bucket[i]; 16188649Sjake } 16288649Sjake } while ((i = (i + 1) & (TSB_BUCKET_SIZE - 1)) != b0); 16380709Sjake 16488649Sjake if (tp == NULL) 16588649Sjake tp = rtp; 16688649Sjake if ((tp->tte_data & TD_V) != 0) { 167108700Sjake PMAP_STATS_INC(tsb_nrepl); 16897027Sjake ova = TTE_GET_VA(tp); 16997447Sjake pmap_remove_tte(pm, NULL, tp, ova); 170100718Sjake tlb_page_demap(pm, ova); 17182903Sjake } 17280709Sjake 17397447Sjakeenter: 174112697Sjake if ((m->flags & PG_FICTITIOUS) == 0) { 175112697Sjake data |= TD_CP; 176224746Skib if ((m->oflags & VPO_UNMANAGED) == 0) { 177112697Sjake pm->pm_stats.resident_count++; 178112697Sjake data |= TD_PV; 179112697Sjake } 180112697Sjake if (pmap_cache_enter(m, va) != 0) 181112697Sjake data |= TD_CV; 182112697Sjake TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 183112697Sjake } else 184112697Sjake data |= TD_FAKE | TD_E; 18597446Sjake 186102040Sjake tp->tte_vpn = TV_VPN(va, sz); 18797030Sjake tp->tte_data = data; 18880709Sjake 18988649Sjake return (tp); 19080709Sjake} 19188826Stmm 19288826Stmm/* 19388826Stmm * Traverse the tsb of a pmap, calling the callback function for any tte entry 19488826Stmm * that has a virtual address between start and end. If this function returns 0, 19588826Stmm * tsb_foreach() terminates. 196162543Salc * This is used by pmap_remove(), pmap_protect(), and pmap_copy() in the case 197162543Salc * that the number of pages in the range given to them reaches the 198162543Salc * dimensions of the tsb size as an optimization. 19988826Stmm */ 20088826Stmmvoid 20191168Sjaketsb_foreach(pmap_t pm1, pmap_t pm2, vm_offset_t start, vm_offset_t end, 20288826Stmm tsb_callback_t *callback) 20388826Stmm{ 20488826Stmm vm_offset_t va; 20588826Stmm struct tte *tp; 206162543Salc struct tte *tsbp; 207162543Salc uintptr_t i; 208162543Salc uintptr_t n; 20988826Stmm 210108700Sjake PMAP_STATS_INC(tsb_nforeach); 211162543Salc if (pm1 == kernel_pmap) { 212162543Salc tsbp = tsb_kernel; 213162543Salc n = tsb_kernel_size / sizeof(struct tte); 214162543Salc } else { 215162543Salc tsbp = pm1->pm_tsb; 216162543Salc n = TSB_SIZE; 217162543Salc } 218162543Salc for (i = 0; i < n; i++) { 219162543Salc tp = &tsbp[i]; 22088826Stmm if ((tp->tte_data & TD_V) != 0) { 22197027Sjake va = TTE_GET_VA(tp); 22288826Stmm if (va >= start && va < end) { 22391168Sjake if (!callback(pm1, pm2, tp, va)) 22488826Stmm break; 22588826Stmm } 22688826Stmm } 22788826Stmm } 22888826Stmm} 229