1/* 2 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org) 4 * Copyright (C) 2000, 2001 Broadcom Corporation 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 */ 20#include <linux/config.h> 21#include <linux/init.h> 22#include <asm/mmu_context.h> 23#include <asm/bootinfo.h> 24#include <asm/cacheops.h> 25#include <asm/cpu.h> 26#include <asm/uaccess.h> 27 28/* These are probed at ld_mmu time */ 29static unsigned long icache_size; 30static unsigned long dcache_size; 31 32static unsigned long icache_line_size; 33static unsigned long dcache_line_size; 34 35static unsigned int icache_index_mask; 36 37static unsigned long icache_assoc; 38static unsigned long dcache_assoc; 39 40static unsigned int icache_sets; 41static unsigned int dcache_sets; 42 43void pgd_init(unsigned long page) 44{ 45 unsigned long *p = (unsigned long *) page; 46 int i; 47 48 for (i = 0; i < USER_PTRS_PER_PGD; i+=8) { 49 p[i + 0] = (unsigned long) invalid_pte_table; 50 p[i + 1] = (unsigned long) invalid_pte_table; 51 p[i + 2] = (unsigned long) invalid_pte_table; 52 p[i + 3] = (unsigned long) invalid_pte_table; 53 p[i + 4] = (unsigned long) invalid_pte_table; 54 p[i + 5] = (unsigned long) invalid_pte_table; 55 p[i + 6] = (unsigned long) invalid_pte_table; 56 p[i + 7] = (unsigned long) invalid_pte_table; 57 } 58} 59 60/* 61 * The dcache is fully coherent to the system, with one 62 * big caveat: the instruction stream. In other words, 63 * if we miss in the icache, and have dirty data in the 64 * L1 dcache, then we'll go out to memory (or the L2) and 65 * get the not-as-recent data. 66 * 67 * So the only time we have to flush the dcache is when 68 * we're flushing the icache. Since the L2 is fully 69 * coherent to everything, including I/O, we never have 70 * to flush it 71 */ 72 73/* 74 * Writeback and invalidate the entire dcache 75 */ 76static void sb1_writeback_inv_dcache_all(void) 77{ 78 /* 79 * Register usage: 80 * 81 * $1 - moving cache index 82 * $2 - set count 83 */ 84 __asm__ __volatile__ ( 85 ".set push \n" 86 ".set noreorder \n" 87 ".set noat \n" 88 ".set mips4 \n" 89 " move $1, %2 \n" /* Start at index 0 */ 90 "1: cache %3, 0($1) \n" /* Invalidate this index */ 91 " daddiu %1, %1, -1 \n" /* Decrement loop count */ 92 " bnez %1, 1b \n" /* loop test */ 93 " daddu $1, $1, %0 \n" /* Next address */ 94 ".set pop \n" 95 : 96 : "r" (dcache_line_size), "r" (dcache_sets * dcache_assoc), 97 "r" (KSEG0), "i" (Index_Writeback_Inv_D)); 98} 99 100 101static inline void __sb1_writeback_inv_dcache_range(unsigned long start, 102 unsigned long end) 103{ 104 __asm__ __volatile__ ( 105 " .set push \n" 106 " .set noreorder \n" 107 " .set noat \n" 108 " .set mips4 \n" 109 "1: cache %3, (0<<13)(%0) \n" /* Index-WB-inval this address */ 110 " cache %3, (1<<13)(%0) \n" /* Index-WB-inval this address */ 111 " cache %3, (2<<13)(%0) \n" /* Index-WB-inval this address */ 112 " cache %3, (3<<13)(%0) \n" /* Index-WB-inval this address */ 113 " xori $1, %0, 1<<12 \n" 114 " cache %3, (0<<13)($1) \n" /* Index-WB-inval this address */ 115 " cache %3, (1<<13)($1) \n" /* Index-WB-inval this address */ 116 " cache %3, (2<<13)($1) \n" /* Index-WB-inval this address */ 117 " cache %3, (3<<13)($1) \n" /* Index-WB-inval this address */ 118 " bne %0, %1, 1b \n" /* loop test */ 119 " addu %0, %0, %2 \n" /* next line */ 120 " sync \n" 121 " .set pop \n" 122 : 123 : "r" (start & ~(dcache_line_size - 1)), 124 "r" ((end - 1) & ~(dcache_line_size - 1)), 125 "r" (dcache_line_size), 126 "i" (Index_Writeback_Inv_D)); 127} 128 129 130static inline void local_sb1___flush_icache_all(void) 131{ 132 __asm__ __volatile__ ( 133 ".set push \n" 134 ".set noreorder \n" 135 ".set noat \n" 136 ".set mips4 \n" 137 " move $1, %2 \n" /* Start at index 0 */ 138 "1: cache %3, 0($1) \n" /* Invalidate this index */ 139 " daddiu %1, %1, -1 \n" /* Decrement loop count */ 140 " bnez %1, 1b \n" /* loop test */ 141 " daddu $1, $1, %0 \n" /* Next address */ 142 ".set pop \n" 143 : 144 : "r" (icache_line_size), "r" (icache_sets * icache_assoc), 145 "r" (KSEG0), "i" (Index_Invalidate_I)); 146} 147 148static void local_sb1___flush_cache_all(void) 149{ 150 sb1_writeback_inv_dcache_all(); 151 local_sb1___flush_icache_all(); 152} 153 154#ifdef CONFIG_SMP 155extern void sb1___flush_cache_all_ipi(void *ignored); 156asm("sb1___flush_cache_all_ipi = local_sb1___flush_cache_all"); 157 158static void sb1___flush_cache_all(void) 159{ 160 smp_call_function(sb1___flush_cache_all_ipi, 0, 1, 1); 161 local_sb1___flush_cache_all(); 162} 163#else 164extern void sb1___flush_cache_all(void); 165asm("sb1___flush_cache_all = local_sb1___flush_cache_all"); 166#endif 167 168static void sb1_flush_icache_all(void) 169{ 170 sb1_writeback_inv_dcache_all(); 171 local_sb1___flush_icache_all(); 172} 173 174/* 175 * When flushing a range in the icache, we have to first writeback 176 * the dcache for the same range, so new ifetches will see any 177 * data that was dirty in the dcache. 178 * 179 * The start/end arguments are expected to be Kseg addresses. 180 */ 181 182static void local_sb1_flush_icache_range(unsigned long start, 183 unsigned long end) 184{ 185 /* 186 * Don't do ridiculously large flushes; if it's more than 2x the cache 187 * size, it's probably going to be faster to just flush the whole thing. 188 * 189 * When time permits (Ha!) this x2 factor should be quantified more 190 * formally. 191 */ 192 if ((end - start) > (icache_size * 2)) { 193 sb1_flush_icache_all(); 194 return; 195 } 196 197 __sb1_writeback_inv_dcache_range(start, end); 198 199 __asm__ __volatile__ ( 200 ".set push \n" 201 ".set noreorder \n" 202 ".set noat \n" 203 ".set mips4 \n" 204 " move $1, %0 \n" 205 ".align 3 \n" 206 "1: cache %3, (0<<13)($1) \n" /* Index-inval this address */ 207 " cache %3, (1<<13)($1) \n" /* Index-inval this address */ 208 " cache %3, (2<<13)($1) \n" /* Index-inval this address */ 209 " cache %3, (3<<13)($1) \n" /* Index-inval this address */ 210 " bne $1, %1, 1b \n" /* loop test */ 211 " addu $1, $1, %2 \n" /* next line */ 212 " sync \n" 213 ".set pop \n" 214 : 215 : "r" (start & ~(icache_line_size - 1)), 216 "r" ((end - 1) & ~(dcache_line_size - 1)), 217 "r" (icache_line_size), 218 "i" (Index_Invalidate_I)); 219} 220 221#ifdef CONFIG_SMP 222struct flush_icache_range_args { 223 unsigned long start; 224 unsigned long end; 225}; 226 227static void sb1_flush_icache_range_ipi(void *info) 228{ 229 struct flush_icache_range_args *args = info; 230 231 local_sb1_flush_icache_range(args->start, args->end); 232} 233 234void sb1_flush_icache_range(unsigned long start, unsigned long end) 235{ 236 struct flush_icache_range_args args; 237 238 args.start = start; 239 args.end = end; 240 smp_call_function(sb1_flush_icache_range_ipi, &args, 1, 1); 241 local_sb1_flush_icache_range(start, end); 242} 243#else 244void sb1_flush_icache_range(unsigned long start, unsigned long end); 245asm("sb1_flush_icache_range = local_sb1_flush_icache_range"); 246#endif 247 248/* 249 * If the page isn't executable, no icache flush is needed 250 */ 251static void sb1_flush_icache_page(struct vm_area_struct *vma, 252 struct page *page) 253{ 254 if (!(vma->vm_flags & VM_EXEC)) { 255 return; 256 } 257 258 /* 259 * We're not sure of the virtual address(es) involved here, so 260 * conservatively flush the entire caches on all processors 261 * (ouch). 262 * 263 * Bumping the ASID may well be cheaper, need to experiment ... 264 */ 265 sb1___flush_cache_all(); 266} 267 268/* 269 * A signal trampoline must fit into a single cacheline. 270 */ 271static void local_sb1_flush_cache_sigtramp(unsigned long addr) 272{ 273 /* 274 * This routine is called on both cores. We assume the ASID 275 * has been set up properly, and interrupts are off to prevent 276 * reschedule and TLB changes. 277 */ 278 __asm__ __volatile__ ( 279 " .set push \n" 280 " .set noreorder \n" 281 " .set noat \n" 282 " .set mips4 \n" 283 " cache %2, (0<<13)(%0) \n" /* Index-inval this address */ 284 " cache %2, (1<<13)(%0) \n" /* Index-inval this address */ 285 " cache %2, (2<<13)(%0) \n" /* Index-inval this address */ 286 " cache %2, (3<<13)(%0) \n" /* Index-inval this address */ 287 " xor $1, %0, 1<<12 \n" /* Flip index bit 12 */ 288 " cache %2, (0<<13)($1) \n" /* Index-inval this address */ 289 " cache %2, (1<<13)($1) \n" /* Index-inval this address */ 290 " cache %2, (2<<13)($1) \n" /* Index-inval this address */ 291 " cache %2, (3<<13)($1) \n" /* Index-inval this address */ 292 " cache %3, (0<<13)(%0) \n" /* Index-inval this address */ 293 " cache %3, (1<<13)(%0) \n" /* Index-inval this address */ 294 " cache %3, (2<<13)(%0) \n" /* Index-inval this address */ 295 " cache %3, (3<<13)(%0) \n" /* Index-inval this address */ 296 " .set pop \n" 297 : "=r" (addr) 298 : "0" (addr), "i" (Index_Writeback_Inv_D), "i" (Index_Invalidate_I)); 299} 300 301#ifdef CONFIG_SMP 302static void sb1_flush_cache_sigtramp_ipi(void *info) 303{ 304 unsigned long iaddr = (unsigned long) info; 305 local_sb1_flush_cache_sigtramp(iaddr); 306} 307 308static void sb1_flush_cache_sigtramp(unsigned long addr) 309{ 310 local_sb1_flush_cache_sigtramp(addr); 311 smp_call_function(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1); 312} 313#else 314void sb1_flush_cache_sigtramp(unsigned long addr); 315asm("sb1_flush_cache_sigtramp = local_sb1_flush_cache_sigtramp"); 316#endif 317 318/* 319 * For executable pages, we need to writeback any local dirty data in 320 * the dcache and invalidate the icache 321 * 322 * XXXKW the dcache flush on the remote core may be argued to 323 * be unnecessary. 324 */ 325static void local_sb1_flush_cache_page(struct vm_area_struct *vma, 326 unsigned long addr) 327{ 328 if (!(vma->vm_flags & VM_EXEC)) 329 return; 330 331 addr &= PAGE_MASK; 332 local_sb1_flush_icache_range(addr, addr + PAGE_SIZE); 333} 334 335#ifdef CONFIG_SMP 336struct flush_cache_page_struct 337{ 338 struct vm_area_struct *vma; 339 unsigned long page; 340}; 341 342static void sb1_flush_cache_page_ipi(void *info) 343{ 344 struct flush_cache_page_struct *args = 345 (struct flush_cache_page_struct *)info; 346 local_sb1_flush_cache_page(args->vma, args->page); 347} 348 349static void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr) 350{ 351 struct flush_cache_page_struct args; 352 args.vma = vma; 353 args.page = addr; 354 local_sb1_flush_cache_page(vma, addr); 355 smp_call_function(sb1_flush_cache_page_ipi, (void *) &args, 1, 1); 356} 357#else 358void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr); 359asm("sb1_flush_cache_page = local_sb1_flush_cache_page"); 360#endif 361 362/* 363 * Anything that just flushes dcache state can be ignored, as we're always 364 * coherent in dcache space. This is just a dummy function that all the 365 * nop'ed routines point to 366 */ 367static void sb1_nop(void) 368{ 369} 370 371/* 372 * Cache set values (from the mips64 spec) 373 * 0 - 64 374 * 1 - 128 375 * 2 - 256 376 * 3 - 512 377 * 4 - 1024 378 * 5 - 2048 379 * 6 - 4096 380 * 7 - Reserved 381 */ 382 383static unsigned int decode_cache_sets(unsigned int config_field) 384{ 385 if (config_field == 7) { 386 /* JDCXXX - Find a graceful way to abort. */ 387 return 0; 388 } 389 return (1<<(config_field + 6)); 390} 391 392/* 393 * Cache line size values (from the mips64 spec) 394 * 0 - No cache present. 395 * 1 - 4 bytes 396 * 2 - 8 bytes 397 * 3 - 16 bytes 398 * 4 - 32 bytes 399 * 5 - 64 bytes 400 * 6 - 128 bytes 401 * 7 - Reserved 402 */ 403 404static unsigned int decode_cache_line_size(unsigned int config_field) 405{ 406 if (config_field == 0) { 407 return 0; 408 } else if (config_field == 7) { 409 /* JDCXXX - Find a graceful way to abort. */ 410 return 0; 411 } 412 return (1<<(config_field + 1)); 413} 414 415/* 416 * Relevant bits of the config1 register format (from the MIPS32/MIPS64 specs) 417 * 418 * 24:22 Icache sets per way 419 * 21:19 Icache line size 420 * 18:16 Icache Associativity 421 * 15:13 Dcache sets per way 422 * 12:10 Dcache line size 423 * 9:7 Dcache Associativity 424 */ 425 426static __init void probe_cache_sizes(void) 427{ 428 u32 config1; 429 430 config1 = read_c0_config1(); 431 icache_line_size = decode_cache_line_size((config1 >> 19) & 0x7); 432 dcache_line_size = decode_cache_line_size((config1 >> 10) & 0x7); 433 icache_sets = decode_cache_sets((config1 >> 22) & 0x7); 434 dcache_sets = decode_cache_sets((config1 >> 13) & 0x7); 435 icache_assoc = ((config1 >> 16) & 0x7) + 1; 436 dcache_assoc = ((config1 >> 7) & 0x7) + 1; 437 icache_size = icache_line_size * icache_sets * icache_assoc; 438 dcache_size = dcache_line_size * dcache_sets * dcache_assoc; 439 icache_index_mask = (icache_sets - 1) * icache_line_size; 440} 441 442/* 443 * This is called from loadmmu.c. We have to set up all the 444 * memory management function pointers, as well as initialize 445 * the caches and tlbs 446 */ 447void ld_mmu_sb1(void) 448{ 449 unsigned long temp; 450 451#ifdef CONFIG_SB1_CACHE_ERROR 452 /* Special cache error handler for SB1 */ 453 extern char except_vec2_sb1; 454 455 memcpy((void *)(KSEG0 + 0x100), &except_vec2_sb1, 0x80); 456 memcpy((void *)(KSEG1 + 0x100), &except_vec2_sb1, 0x80); 457#endif 458 459 probe_cache_sizes(); 460 461 _clear_page = sb1_clear_page; 462 _copy_page = sb1_copy_page; 463 464 /* None of these are needed for the sb1 */ 465 _flush_cache_mm = (void (*)(struct mm_struct *))sb1_nop; 466 _flush_cache_range = (void *) sb1_nop; 467 _flush_cache_all = sb1_nop; 468 469 /* 470 * "flush_page_to_ram" is expected to prevent virtual aliasing 471 * in the Dcache, and is called before a new mapping for a 472 * page is about the be installed. Since our Dcache is 473 * physically indexed and tagged, there can't be aliasing. If 474 * coherence with I-stream is needed, an icache will be used 475 * -- so we don't have to do any flushing. 476 */ 477 _flush_page_to_ram = (void (*)(struct page *)) sb1_nop; 478 479 ___flush_cache_all = sb1___flush_cache_all; 480 _flush_icache_page = sb1_flush_icache_page; 481 _flush_icache_range = sb1_flush_icache_range; 482 _flush_cache_page = sb1_flush_cache_page; 483 _flush_cache_sigtramp = sb1_flush_cache_sigtramp; 484 _flush_icache_all = sb1_flush_icache_all; 485 486 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 487 /* 488 * This is the only way to force the update of K0 to complete 489 * before subsequent instruction fetch. 490 */ 491 __asm__ __volatile__ ( 492 " .set push \n" 493 " .set mips4 \n" 494 " la %0, 1f \n" 495 " mtc0 %0, $14 \n" 496 " eret \n" 497 "1: .set pop \n" 498 : "=r" (temp)); 499 flush_cache_all(); 500} 501