1/* $Id: init.c,v 1.1.1.1 2008/10/15 03:26:19 james26_jang Exp $ 2 * arch/sparc64/mm/init.c 3 * 4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu) 5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 6 */ 7 8#include <linux/config.h> 9#include <linux/kernel.h> 10#include <linux/sched.h> 11#include <linux/string.h> 12#include <linux/init.h> 13#include <linux/bootmem.h> 14#include <linux/mm.h> 15#include <linux/slab.h> 16#include <linux/blk.h> 17#include <linux/swap.h> 18#include <linux/swapctl.h> 19#include <linux/pagemap.h> 20#include <linux/fs.h> 21#include <linux/seq_file.h> 22 23#include <asm/head.h> 24#include <asm/system.h> 25#include <asm/page.h> 26#include <asm/pgalloc.h> 27#include <asm/pgtable.h> 28#include <asm/oplib.h> 29#include <asm/iommu.h> 30#include <asm/io.h> 31#include <asm/uaccess.h> 32#include <asm/mmu_context.h> 33#include <asm/dma.h> 34#include <asm/starfire.h> 35#include <asm/tlb.h> 36#include <asm/spitfire.h> 37 38mmu_gather_t mmu_gathers[NR_CPUS]; 39 40extern void device_scan(void); 41 42struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS]; 43 44unsigned long *sparc64_valid_addr_bitmap; 45 46/* Ugly, but necessary... -DaveM */ 47unsigned long phys_base; 48 49enum ultra_tlb_layout tlb_type = spitfire; 50 51/* get_new_mmu_context() uses "cache + 1". */ 52spinlock_t ctx_alloc_lock = SPIN_LOCK_UNLOCKED; 53unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1; 54#define CTX_BMAP_SLOTS (1UL << (CTX_VERSION_SHIFT - 6)) 55unsigned long mmu_context_bmap[CTX_BMAP_SLOTS]; 56 57/* References to section boundaries */ 58extern char __init_begin, __init_end, _start, _end, etext, edata; 59 60/* Initial ramdisk setup */ 61extern unsigned int sparc_ramdisk_image; 62extern unsigned int sparc_ramdisk_size; 63 64struct page *mem_map_zero; 65 66int bigkernel = 0; 67 68int do_check_pgt_cache(int low, int high) 69{ 70 int freed = 0; 71 72 if (pgtable_cache_size > high) { 73 do { 74#ifdef CONFIG_SMP 75 if (pgd_quicklist) 76 free_pgd_slow(get_pgd_fast()), freed++; 77#endif 78 if (pte_quicklist[0]) 79 free_pte_slow(pte_alloc_one_fast(NULL, 0)), freed++; 80 if (pte_quicklist[1]) 81 free_pte_slow(pte_alloc_one_fast(NULL, 1 << (PAGE_SHIFT + 10))), freed++; 82 } while (pgtable_cache_size > low); 83 } 84#ifndef CONFIG_SMP 85 if (pgd_cache_size > high / 4) { 86 struct page *page, *page2; 87 for (page2 = NULL, page = (struct page *)pgd_quicklist; page;) { 88 if ((unsigned long)page->pprev_hash == 3) { 89 if (page2) 90 page2->next_hash = page->next_hash; 91 else 92 (struct page *)pgd_quicklist = page->next_hash; 93 page->next_hash = NULL; 94 page->pprev_hash = NULL; 95 pgd_cache_size -= 2; 96 __free_page(page); 97 freed++; 98 if (page2) 99 page = page2->next_hash; 100 else 101 page = (struct page *)pgd_quicklist; 102 if (pgd_cache_size <= low / 4) 103 break; 104 continue; 105 } 106 page2 = page; 107 page = page->next_hash; 108 } 109 } 110#endif 111 return freed; 112} 113 114extern void __update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t); 115 116#ifdef CONFIG_DEBUG_DCFLUSH 117atomic_t dcpage_flushes = ATOMIC_INIT(0); 118#ifdef CONFIG_SMP 119atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0); 120#endif 121#endif 122 123__inline__ void flush_dcache_page_impl(struct page *page) 124{ 125#ifdef CONFIG_DEBUG_DCFLUSH 126 atomic_inc(&dcpage_flushes); 127#endif 128 129#if (L1DCACHE_SIZE > PAGE_SIZE) 130 __flush_dcache_page(page->virtual, 131 ((tlb_type == spitfire) && 132 page->mapping != NULL)); 133#else 134 if (page->mapping != NULL && 135 tlb_type == spitfire) 136 __flush_icache_page(__pa(page->virtual)); 137#endif 138} 139 140#define PG_dcache_dirty PG_arch_1 141 142#define dcache_dirty_cpu(page) \ 143 (((page)->flags >> 24) & (NR_CPUS - 1UL)) 144 145static __inline__ void set_dcache_dirty(struct page *page) 146{ 147 unsigned long mask = smp_processor_id(); 148 unsigned long non_cpu_bits = (1UL << 24UL) - 1UL; 149 mask = (mask << 24) | (1UL << PG_dcache_dirty); 150 __asm__ __volatile__("1:\n\t" 151 "ldx [%2], %%g7\n\t" 152 "and %%g7, %1, %%g5\n\t" 153 "or %%g5, %0, %%g5\n\t" 154 "casx [%2], %%g7, %%g5\n\t" 155 "cmp %%g7, %%g5\n\t" 156 "bne,pn %%xcc, 1b\n\t" 157 " membar #StoreLoad | #StoreStore" 158 : /* no outputs */ 159 : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags) 160 : "g5", "g7"); 161} 162 163static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu) 164{ 165 unsigned long mask = (1UL << PG_dcache_dirty); 166 167 __asm__ __volatile__("! test_and_clear_dcache_dirty\n" 168 "1:\n\t" 169 "ldx [%2], %%g7\n\t" 170 "srlx %%g7, 24, %%g5\n\t" 171 "cmp %%g5, %0\n\t" 172 "bne,pn %%icc, 2f\n\t" 173 " andn %%g7, %1, %%g5\n\t" 174 "casx [%2], %%g7, %%g5\n\t" 175 "cmp %%g7, %%g5\n\t" 176 "bne,pn %%xcc, 1b\n\t" 177 " membar #StoreLoad | #StoreStore\n" 178 "2:" 179 : /* no outputs */ 180 : "r" (cpu), "r" (mask), "r" (&page->flags) 181 : "g5", "g7"); 182} 183 184void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte) 185{ 186 struct page *page = pte_page(pte); 187 unsigned long pg_flags; 188 189 if (VALID_PAGE(page) && 190 page->mapping && 191 ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) { 192 int cpu = (pg_flags >> 24); 193 194 /* This is just to optimize away some function calls 195 * in the SMP case. 196 */ 197 if (cpu == smp_processor_id()) 198 flush_dcache_page_impl(page); 199 else 200 smp_flush_dcache_page_impl(page, cpu); 201 202 clear_dcache_dirty_cpu(page, cpu); 203 } 204 __update_mmu_cache(vma, address, pte); 205} 206 207void flush_dcache_page(struct page *page) 208{ 209 int dirty = test_bit(PG_dcache_dirty, &page->flags); 210 int dirty_cpu = dcache_dirty_cpu(page); 211 212 if (page->mapping && 213 page->mapping->i_mmap == NULL && 214 page->mapping->i_mmap_shared == NULL) { 215 if (dirty) { 216 if (dirty_cpu == smp_processor_id()) 217 return; 218 smp_flush_dcache_page_impl(page, dirty_cpu); 219 } 220 set_dcache_dirty(page); 221 } else { 222 /* We could delay the flush for the !page->mapping 223 * case too. But that case is for exec env/arg 224 * pages and those are %99 certainly going to get 225 * faulted into the tlb (and thus flushed) anyways. 226 */ 227 flush_dcache_page_impl(page); 228 } 229} 230 231void flush_icache_range(unsigned long start, unsigned long end) 232{ 233 /* Cheetah has coherent I-cache. */ 234 if (tlb_type == spitfire) { 235 unsigned long kaddr; 236 237 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) 238 __flush_icache_page(__get_phys(kaddr)); 239 } 240} 241 242void show_mem(void) 243{ 244 printk("Mem-info:\n"); 245 show_free_areas(); 246 printk("Free swap: %6dkB\n", 247 nr_swap_pages << (PAGE_SHIFT-10)); 248 printk("%ld pages of RAM\n", num_physpages); 249 printk("%d free pages\n", nr_free_pages()); 250 printk("%d pages in page table cache\n",pgtable_cache_size); 251#ifndef CONFIG_SMP 252 printk("%d entries in page dir cache\n",pgd_cache_size); 253#endif 254 show_buffers(); 255} 256 257void mmu_info(struct seq_file *m) 258{ 259 if (tlb_type == cheetah) 260 seq_printf(m, "MMU Type\t: Cheetah\n"); 261 else if (tlb_type == cheetah_plus) 262 seq_printf(m, "MMU Type\t: Cheetah+\n"); 263 else if (tlb_type == spitfire) 264 seq_printf(m, "MMU Type\t: Spitfire\n"); 265 else 266 seq_printf(m, "MMU Type\t: ???\n"); 267 268#ifdef CONFIG_DEBUG_DCFLUSH 269 seq_printf(m, "DCPageFlushes\t: %d\n", 270 atomic_read(&dcpage_flushes)); 271#ifdef CONFIG_SMP 272 seq_printf(m, "DCPageFlushesXC\t: %d\n", 273 atomic_read(&dcpage_flushes_xcall)); 274#endif /* CONFIG_SMP */ 275#endif /* CONFIG_DEBUG_DCFLUSH */ 276} 277 278struct linux_prom_translation { 279 unsigned long virt; 280 unsigned long size; 281 unsigned long data; 282}; 283 284extern unsigned long prom_boot_page; 285extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle); 286extern int prom_get_mmu_ihandle(void); 287extern void register_prom_callbacks(void); 288 289/* Exported for SMP bootup purposes. */ 290unsigned long kern_locked_tte_data; 291 292void __init early_pgtable_allocfail(char *type) 293{ 294 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type); 295 prom_halt(); 296} 297 298#define BASE_PAGE_SIZE 8192 299static pmd_t *prompmd; 300 301/* 302 * Translate PROM's mapping we capture at boot time into physical address. 303 * The second parameter is only set from prom_callback() invocations. 304 */ 305unsigned long prom_virt_to_phys(unsigned long promva, int *error) 306{ 307 pmd_t *pmdp = prompmd + ((promva >> 23) & 0x7ff); 308 pte_t *ptep; 309 unsigned long base; 310 311 if (pmd_none(*pmdp)) { 312 if (error) 313 *error = 1; 314 return(0); 315 } 316 ptep = (pte_t *)pmd_page(*pmdp) + ((promva >> 13) & 0x3ff); 317 if (!pte_present(*ptep)) { 318 if (error) 319 *error = 1; 320 return(0); 321 } 322 if (error) { 323 *error = 0; 324 return(pte_val(*ptep)); 325 } 326 base = pte_val(*ptep) & _PAGE_PADDR; 327 return(base + (promva & (BASE_PAGE_SIZE - 1))); 328} 329 330static void inherit_prom_mappings(void) 331{ 332 struct linux_prom_translation *trans; 333 unsigned long phys_page, tte_vaddr, tte_data; 334 void (*remap_func)(unsigned long, unsigned long, int); 335 pmd_t *pmdp; 336 pte_t *ptep; 337 int node, n, i, tsz; 338 extern unsigned int obp_iaddr_patch[2], obp_daddr_patch[2]; 339 340 node = prom_finddevice("/virtual-memory"); 341 n = prom_getproplen(node, "translations"); 342 if (n == 0 || n == -1) { 343 prom_printf("Couldn't get translation property\n"); 344 prom_halt(); 345 } 346 n += 5 * sizeof(struct linux_prom_translation); 347 for (tsz = 1; tsz < n; tsz <<= 1) 348 /* empty */; 349 trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, 0UL); 350 if (trans == NULL) { 351 prom_printf("inherit_prom_mappings: Cannot alloc translations.\n"); 352 prom_halt(); 353 } 354 memset(trans, 0, tsz); 355 356 if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) { 357 prom_printf("Couldn't get translation property\n"); 358 prom_halt(); 359 } 360 n = n / sizeof(*trans); 361 362 /* 363 * The obp translations are saved based on 8k pagesize, since obp can use 364 * a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000, ie obp 365 * range, are handled in entry.S and do not use the vpte scheme (see rant 366 * in inherit_locked_prom_mappings()). 367 */ 368#define OBP_PMD_SIZE 2048 369 prompmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, 0UL); 370 if (prompmd == NULL) 371 early_pgtable_allocfail("pmd"); 372 memset(prompmd, 0, OBP_PMD_SIZE); 373 for (i = 0; i < n; i++) { 374 unsigned long vaddr; 375 376 if (trans[i].virt >= LOW_OBP_ADDRESS && trans[i].virt < HI_OBP_ADDRESS) { 377 for (vaddr = trans[i].virt; 378 ((vaddr < trans[i].virt + trans[i].size) && 379 (vaddr < HI_OBP_ADDRESS)); 380 vaddr += BASE_PAGE_SIZE) { 381 unsigned long val; 382 383 pmdp = prompmd + ((vaddr >> 23) & 0x7ff); 384 if (pmd_none(*pmdp)) { 385 ptep = __alloc_bootmem(BASE_PAGE_SIZE, 386 BASE_PAGE_SIZE, 387 0UL); 388 if (ptep == NULL) 389 early_pgtable_allocfail("pte"); 390 memset(ptep, 0, BASE_PAGE_SIZE); 391 pmd_set(pmdp, ptep); 392 } 393 ptep = (pte_t *)pmd_page(*pmdp) + 394 ((vaddr >> 13) & 0x3ff); 395 396 val = trans[i].data; 397 398 /* Clear diag TTE bits. */ 399 if (tlb_type == spitfire) 400 val &= ~0x0003fe0000000000UL; 401 402 set_pte (ptep, __pte(val | _PAGE_MODIFIED)); 403 trans[i].data += BASE_PAGE_SIZE; 404 } 405 } 406 } 407 phys_page = __pa(prompmd); 408 obp_iaddr_patch[0] |= (phys_page >> 10); 409 obp_iaddr_patch[1] |= (phys_page & 0x3ff); 410 flushi((long)&obp_iaddr_patch[0]); 411 obp_daddr_patch[0] |= (phys_page >> 10); 412 obp_daddr_patch[1] |= (phys_page & 0x3ff); 413 flushi((long)&obp_daddr_patch[0]); 414 415 /* Now fixup OBP's idea about where we really are mapped. */ 416 prom_printf("Remapping the kernel... "); 417 418 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 419 "flush %%g6" 420 : /* No outputs */ 421 : "r" (0), 422 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 423 424 switch (tlb_type) { 425 default: 426 case spitfire: 427 phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()); 428 break; 429 430 case cheetah: 431 case cheetah_plus: 432 phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent()); 433 break; 434 }; 435 436 phys_page &= _PAGE_PADDR; 437 phys_page += ((unsigned long)&prom_boot_page - 438 (unsigned long)KERNBASE); 439 440 if (tlb_type == spitfire) { 441 /* Lock this into i/d tlb entry 59 */ 442 __asm__ __volatile__( 443 "stxa %%g0, [%2] %3\n\t" 444 "stxa %0, [%1] %4\n\t" 445 "membar #Sync\n\t" 446 "flush %%g6\n\t" 447 "stxa %%g0, [%2] %5\n\t" 448 "stxa %0, [%1] %6\n\t" 449 "membar #Sync\n\t" 450 "flush %%g6" 451 : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP | 452 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), 453 "r" (59 << 3), "r" (TLB_TAG_ACCESS), 454 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), 455 "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS) 456 : "memory"); 457 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 458 /* Lock this into i/d tlb-0 entry 11 */ 459 __asm__ __volatile__( 460 "stxa %%g0, [%2] %3\n\t" 461 "stxa %0, [%1] %4\n\t" 462 "membar #Sync\n\t" 463 "flush %%g6\n\t" 464 "stxa %%g0, [%2] %5\n\t" 465 "stxa %0, [%1] %6\n\t" 466 "membar #Sync\n\t" 467 "flush %%g6" 468 : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP | 469 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), 470 "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS), 471 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), 472 "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS) 473 : "memory"); 474 } else { 475 /* Implement me :-) */ 476 BUG(); 477 } 478 479 tte_vaddr = (unsigned long) KERNBASE; 480 481 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 482 "flush %%g6" 483 : /* No outputs */ 484 : "r" (0), 485 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 486 487 if (tlb_type == spitfire) 488 tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()); 489 else 490 tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent()); 491 492 kern_locked_tte_data = tte_data; 493 494 remap_func = (void *) ((unsigned long) &prom_remap - 495 (unsigned long) &prom_boot_page); 496 497 498 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 499 "flush %%g6" 500 : /* No outputs */ 501 : "r" (0), 502 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 503 504 remap_func((tlb_type == spitfire ? 505 (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) : 506 (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)), 507 (unsigned long) KERNBASE, 508 prom_get_mmu_ihandle()); 509 510 if (bigkernel) 511 remap_func(((tte_data + 0x400000) & _PAGE_PADDR), 512 (unsigned long) KERNBASE + 0x400000, prom_get_mmu_ihandle()); 513 514 /* Flush out that temporary mapping. */ 515 spitfire_flush_dtlb_nucleus_page(0x0); 516 spitfire_flush_itlb_nucleus_page(0x0); 517 518 /* Now lock us back into the TLBs via OBP. */ 519 prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr); 520 prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr); 521 if (bigkernel) { 522 prom_dtlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000, 523 tte_vaddr + 0x400000); 524 prom_itlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000, 525 tte_vaddr + 0x400000); 526 } 527 528 /* Re-read translations property. */ 529 if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) { 530 prom_printf("Couldn't get translation property\n"); 531 prom_halt(); 532 } 533 n = n / sizeof(*trans); 534 535 for (i = 0; i < n; i++) { 536 unsigned long vaddr = trans[i].virt; 537 unsigned long size = trans[i].size; 538 539 if (vaddr < 0xf0000000UL) { 540 unsigned long avoid_start = (unsigned long) KERNBASE; 541 unsigned long avoid_end = avoid_start + (4 * 1024 * 1024); 542 543 if (bigkernel) 544 avoid_end += (4 * 1024 * 1024); 545 if (vaddr < avoid_start) { 546 unsigned long top = vaddr + size; 547 548 if (top > avoid_start) 549 top = avoid_start; 550 prom_unmap(top - vaddr, vaddr); 551 } 552 if ((vaddr + size) > avoid_end) { 553 unsigned long bottom = vaddr; 554 555 if (bottom < avoid_end) 556 bottom = avoid_end; 557 prom_unmap((vaddr + size) - bottom, bottom); 558 } 559 } 560 } 561 562 prom_printf("done.\n"); 563 564 register_prom_callbacks(); 565} 566 567/* The OBP specifications for sun4u mark 0xfffffffc00000000 and 568 * upwards as reserved for use by the firmware (I wonder if this 569 * will be the same on Cheetah...). We use this virtual address 570 * range for the VPTE table mappings of the nucleus so we need 571 * to zap them when we enter the PROM. -DaveM 572 */ 573static void __flush_nucleus_vptes(void) 574{ 575 unsigned long prom_reserved_base = 0xfffffffc00000000UL; 576 int i; 577 578 /* Only DTLB must be checked for VPTE entries. */ 579 if (tlb_type == spitfire) { 580 for (i = 0; i < 63; i++) { 581 unsigned long tag; 582 583 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 584 "flush %%g6" 585 : /* No outputs */ 586 : "r" (0), 587 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 588 589 tag = spitfire_get_dtlb_tag(i); 590 if (((tag & ~(PAGE_MASK)) == 0) && 591 ((tag & (PAGE_MASK)) >= prom_reserved_base)) { 592 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 593 "membar #Sync" 594 : /* no outputs */ 595 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 596 spitfire_put_dtlb_data(i, 0x0UL); 597 } 598 } 599 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 600 for (i = 0; i < 512; i++) { 601 unsigned long tag = cheetah_get_dtlb_tag(i, 2); 602 603 if ((tag & ~PAGE_MASK) == 0 && 604 (tag & PAGE_MASK) >= prom_reserved_base) { 605 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 606 "membar #Sync" 607 : /* no outputs */ 608 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 609 cheetah_put_dtlb_data(i, 0x0UL, 2); 610 } 611 612 if (tlb_type != cheetah_plus) 613 continue; 614 615 tag = cheetah_get_dtlb_tag(i, 3); 616 617 if ((tag & ~PAGE_MASK) == 0 && 618 (tag & PAGE_MASK) >= prom_reserved_base) { 619 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 620 "membar #Sync" 621 : /* no outputs */ 622 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 623 cheetah_put_dtlb_data(i, 0x0UL, 3); 624 } 625 } 626 } else { 627 /* Implement me :-) */ 628 BUG(); 629 } 630} 631 632static int prom_ditlb_set; 633struct prom_tlb_entry { 634 int tlb_ent; 635 unsigned long tlb_tag; 636 unsigned long tlb_data; 637}; 638struct prom_tlb_entry prom_itlb[16], prom_dtlb[16]; 639 640void prom_world(int enter) 641{ 642 unsigned long pstate; 643 int i; 644 645 if (!enter) 646 set_fs(current->thread.current_ds); 647 648 if (!prom_ditlb_set) 649 return; 650 651 /* Make sure the following runs atomically. */ 652 __asm__ __volatile__("flushw\n\t" 653 "rdpr %%pstate, %0\n\t" 654 "wrpr %0, %1, %%pstate" 655 : "=r" (pstate) 656 : "i" (PSTATE_IE)); 657 658 if (enter) { 659 /* Kick out nucleus VPTEs. */ 660 __flush_nucleus_vptes(); 661 662 /* Install PROM world. */ 663 for (i = 0; i < 16; i++) { 664 if (prom_dtlb[i].tlb_ent != -1) { 665 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 666 "membar #Sync" 667 : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS), 668 "i" (ASI_DMMU)); 669 if (tlb_type == spitfire) 670 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 671 prom_dtlb[i].tlb_data); 672 else if (tlb_type == cheetah || tlb_type == cheetah_plus) 673 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 674 prom_dtlb[i].tlb_data); 675 } 676 if (prom_itlb[i].tlb_ent != -1) { 677 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 678 "membar #Sync" 679 : : "r" (prom_itlb[i].tlb_tag), 680 "r" (TLB_TAG_ACCESS), 681 "i" (ASI_IMMU)); 682 if (tlb_type == spitfire) 683 spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 684 prom_itlb[i].tlb_data); 685 else if (tlb_type == cheetah || tlb_type == cheetah_plus) 686 cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 687 prom_itlb[i].tlb_data); 688 } 689 } 690 } else { 691 for (i = 0; i < 16; i++) { 692 if (prom_dtlb[i].tlb_ent != -1) { 693 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 694 "membar #Sync" 695 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 696 if (tlb_type == spitfire) 697 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL); 698 else 699 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL); 700 } 701 if (prom_itlb[i].tlb_ent != -1) { 702 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 703 "membar #Sync" 704 : : "r" (TLB_TAG_ACCESS), 705 "i" (ASI_IMMU)); 706 if (tlb_type == spitfire) 707 spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL); 708 else 709 cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL); 710 } 711 } 712 } 713 __asm__ __volatile__("wrpr %0, 0, %%pstate" 714 : : "r" (pstate)); 715} 716 717void inherit_locked_prom_mappings(int save_p) 718{ 719 int i; 720 int dtlb_seen = 0; 721 int itlb_seen = 0; 722 723 /* Fucking losing PROM has more mappings in the TLB, but 724 * it (conveniently) fails to mention any of these in the 725 * translations property. The only ones that matter are 726 * the locked PROM tlb entries, so we impose the following 727 * irrecovable rule on the PROM, it is allowed 8 locked 728 * entries in the ITLB and 8 in the DTLB. 729 * 730 * Supposedly the upper 16GB of the address space is 731 * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED 732 * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface 733 * used between the client program and the firmware on sun5 734 * systems to coordinate mmu mappings is also COMPLETELY 735 * UNDOCUMENTED!!!!!! Thanks S(t)un! 736 */ 737 if (save_p) { 738 for (i = 0; i < 16; i++) { 739 prom_itlb[i].tlb_ent = -1; 740 prom_dtlb[i].tlb_ent = -1; 741 } 742 } 743 if (tlb_type == spitfire) { 744 int high = SPITFIRE_HIGHEST_LOCKED_TLBENT - bigkernel; 745 for (i = 0; i < high; i++) { 746 unsigned long data; 747 748 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 749 "flush %%g6" 750 : /* No outputs */ 751 : "r" (0), 752 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 753 754 data = spitfire_get_dtlb_data(i); 755 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) { 756 unsigned long tag; 757 758 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 759 "flush %%g6" 760 : /* No outputs */ 761 : "r" (0), 762 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 763 764 tag = spitfire_get_dtlb_tag(i); 765 if (save_p) { 766 prom_dtlb[dtlb_seen].tlb_ent = i; 767 prom_dtlb[dtlb_seen].tlb_tag = tag; 768 prom_dtlb[dtlb_seen].tlb_data = data; 769 } 770 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 771 "membar #Sync" 772 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 773 spitfire_put_dtlb_data(i, 0x0UL); 774 775 dtlb_seen++; 776 if (dtlb_seen > 15) 777 break; 778 } 779 } 780 781 for (i = 0; i < high; i++) { 782 unsigned long data; 783 784 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 785 "flush %%g6" 786 : /* No outputs */ 787 : "r" (0), 788 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 789 790 data = spitfire_get_itlb_data(i); 791 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) { 792 unsigned long tag; 793 794 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 795 "flush %%g6" 796 : /* No outputs */ 797 : "r" (0), 798 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 799 800 tag = spitfire_get_itlb_tag(i); 801 if (save_p) { 802 prom_itlb[itlb_seen].tlb_ent = i; 803 prom_itlb[itlb_seen].tlb_tag = tag; 804 prom_itlb[itlb_seen].tlb_data = data; 805 } 806 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 807 "membar #Sync" 808 : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU)); 809 spitfire_put_itlb_data(i, 0x0UL); 810 811 itlb_seen++; 812 if (itlb_seen > 15) 813 break; 814 } 815 } 816 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 817 int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel; 818 819 for (i = 0; i < high; i++) { 820 unsigned long data; 821 822 data = cheetah_get_ldtlb_data(i); 823 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) { 824 unsigned long tag; 825 826 tag = cheetah_get_ldtlb_tag(i); 827 if (save_p) { 828 prom_dtlb[dtlb_seen].tlb_ent = i; 829 prom_dtlb[dtlb_seen].tlb_tag = tag; 830 prom_dtlb[dtlb_seen].tlb_data = data; 831 } 832 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 833 "membar #Sync" 834 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 835 cheetah_put_ldtlb_data(i, 0x0UL); 836 837 dtlb_seen++; 838 if (dtlb_seen > 15) 839 break; 840 } 841 } 842 843 for (i = 0; i < high; i++) { 844 unsigned long data; 845 846 data = cheetah_get_litlb_data(i); 847 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) { 848 unsigned long tag; 849 850 tag = cheetah_get_litlb_tag(i); 851 if (save_p) { 852 prom_itlb[itlb_seen].tlb_ent = i; 853 prom_itlb[itlb_seen].tlb_tag = tag; 854 prom_itlb[itlb_seen].tlb_data = data; 855 } 856 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 857 "membar #Sync" 858 : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU)); 859 cheetah_put_litlb_data(i, 0x0UL); 860 861 itlb_seen++; 862 if (itlb_seen > 15) 863 break; 864 } 865 } 866 } else { 867 /* Implement me :-) */ 868 BUG(); 869 } 870 if (save_p) 871 prom_ditlb_set = 1; 872} 873 874/* Give PROM back his world, done during reboots... */ 875void prom_reload_locked(void) 876{ 877 int i; 878 879 for (i = 0; i < 16; i++) { 880 if (prom_dtlb[i].tlb_ent != -1) { 881 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 882 "membar #Sync" 883 : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS), 884 "i" (ASI_DMMU)); 885 if (tlb_type == spitfire) 886 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 887 prom_dtlb[i].tlb_data); 888 else if (tlb_type == cheetah || tlb_type == cheetah_plus) 889 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 890 prom_dtlb[i].tlb_data); 891 } 892 893 if (prom_itlb[i].tlb_ent != -1) { 894 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 895 "membar #Sync" 896 : : "r" (prom_itlb[i].tlb_tag), 897 "r" (TLB_TAG_ACCESS), 898 "i" (ASI_IMMU)); 899 if (tlb_type == spitfire) 900 spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 901 prom_itlb[i].tlb_data); 902 else 903 cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 904 prom_itlb[i].tlb_data); 905 } 906 } 907} 908 909void __flush_dcache_range(unsigned long start, unsigned long end) 910{ 911 unsigned long va; 912 913 if (tlb_type == spitfire) { 914 int n = 0; 915 916 for (va = start; va < end; va += 32) { 917 spitfire_put_dcache_tag(va & 0x3fe0, 0x0); 918 if (++n >= 512) 919 break; 920 } 921 } else { 922 start = __pa(start); 923 end = __pa(end); 924 for (va = start; va < end; va += 32) 925 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 926 "membar #Sync" 927 : /* no outputs */ 928 : "r" (va), 929 "i" (ASI_DCACHE_INVALIDATE)); 930 } 931} 932 933void __flush_cache_all(void) 934{ 935 /* Cheetah should be fine here too. */ 936 if (tlb_type == spitfire) { 937 unsigned long va; 938 939 flushw_all(); 940 for (va = 0; va < (PAGE_SIZE << 1); va += 32) 941 spitfire_put_icache_tag(va, 0x0); 942 __asm__ __volatile__("flush %g6"); 943 } 944} 945 946/* If not locked, zap it. */ 947void __flush_tlb_all(void) 948{ 949 unsigned long pstate; 950 int i; 951 952 __asm__ __volatile__("flushw\n\t" 953 "rdpr %%pstate, %0\n\t" 954 "wrpr %0, %1, %%pstate" 955 : "=r" (pstate) 956 : "i" (PSTATE_IE)); 957 if (tlb_type == spitfire) { 958 for (i = 0; i < 64; i++) { 959 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 960 "flush %%g6" 961 : /* No outputs */ 962 : "r" (0), 963 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 964 965 if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) { 966 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 967 "membar #Sync" 968 : /* no outputs */ 969 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); 970 spitfire_put_dtlb_data(i, 0x0UL); 971 } 972 973 __asm__ __volatile__("stxa %0, [%1] %2\n\t" 974 "flush %%g6" 975 : /* No outputs */ 976 : "r" (0), 977 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); 978 979 if (!(spitfire_get_itlb_data(i) & _PAGE_L)) { 980 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 981 "membar #Sync" 982 : /* no outputs */ 983 : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU)); 984 spitfire_put_itlb_data(i, 0x0UL); 985 } 986 } 987 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 988 cheetah_flush_dtlb_all(); 989 cheetah_flush_itlb_all(); 990 } 991 __asm__ __volatile__("wrpr %0, 0, %%pstate" 992 : : "r" (pstate)); 993} 994 995/* Caller does TLB context flushing on local CPU if necessary. 996 * The caller also ensures that CTX_VALID(mm->context) is false. 997 * 998 * We must be careful about boundary cases so that we never 999 * let the user have CTX 0 (nucleus) or we ever use a CTX 1000 * version of zero (and thus NO_CONTEXT would not be caught 1001 * by version mis-match tests in mmu_context.h). 1002 */ 1003void get_new_mmu_context(struct mm_struct *mm) 1004{ 1005 unsigned long ctx, new_ctx; 1006 1007 spin_lock(&ctx_alloc_lock); 1008 ctx = CTX_HWBITS(tlb_context_cache + 1); 1009 new_ctx = find_next_zero_bit(mmu_context_bmap, 1UL << CTX_VERSION_SHIFT, ctx); 1010 if (new_ctx >= (1UL << CTX_VERSION_SHIFT)) { 1011 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1); 1012 if (new_ctx >= ctx) { 1013 int i; 1014 new_ctx = (tlb_context_cache & CTX_VERSION_MASK) + 1015 CTX_FIRST_VERSION; 1016 if (new_ctx == 1) 1017 new_ctx = CTX_FIRST_VERSION; 1018 1019 /* Don't call memset, for 16 entries that's just 1020 * plain silly... 1021 */ 1022 mmu_context_bmap[0] = 3; 1023 mmu_context_bmap[1] = 0; 1024 mmu_context_bmap[2] = 0; 1025 mmu_context_bmap[3] = 0; 1026 for (i = 4; i < CTX_BMAP_SLOTS; i += 4) { 1027 mmu_context_bmap[i + 0] = 0; 1028 mmu_context_bmap[i + 1] = 0; 1029 mmu_context_bmap[i + 2] = 0; 1030 mmu_context_bmap[i + 3] = 0; 1031 } 1032 goto out; 1033 } 1034 } 1035 mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63)); 1036 new_ctx |= (tlb_context_cache & CTX_VERSION_MASK); 1037out: 1038 tlb_context_cache = new_ctx; 1039 spin_unlock(&ctx_alloc_lock); 1040 1041 mm->context = new_ctx; 1042} 1043 1044#ifndef CONFIG_SMP 1045struct pgtable_cache_struct pgt_quicklists; 1046#endif 1047 1048/* OK, we have to color these pages. The page tables are accessed 1049 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S 1050 * code, as well as by PAGE_OFFSET range direct-mapped addresses by 1051 * other parts of the kernel. By coloring, we make sure that the tlbmiss 1052 * fast handlers do not get data from old/garbage dcache lines that 1053 * correspond to an old/stale virtual address (user/kernel) that 1054 * previously mapped the pagetable page while accessing vpte range 1055 * addresses. The idea is that if the vpte color and PAGE_OFFSET range 1056 * color is the same, then when the kernel initializes the pagetable 1057 * using the later address range, accesses with the first address 1058 * range will see the newly initialized data rather than the garbage. 1059 */ 1060#if (L1DCACHE_SIZE > PAGE_SIZE) /* is there D$ aliasing problem */ 1061#define DC_ALIAS_SHIFT 1 1062#else 1063#define DC_ALIAS_SHIFT 0 1064#endif 1065pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address) 1066{ 1067 struct page *page = alloc_pages(GFP_KERNEL, DC_ALIAS_SHIFT); 1068 unsigned long color = VPTE_COLOR(address); 1069 1070 if (page) { 1071 unsigned long *to_free; 1072 unsigned long paddr; 1073 pte_t *pte; 1074 1075#if (L1DCACHE_SIZE > PAGE_SIZE) /* is there D$ aliasing problem */ 1076 set_page_count((page + 1), 1); 1077#endif 1078 paddr = (unsigned long) page_address(page); 1079 memset((char *)paddr, 0, (PAGE_SIZE << DC_ALIAS_SHIFT)); 1080 1081 if (!color) { 1082 pte = (pte_t *) paddr; 1083 to_free = (unsigned long *) (paddr + PAGE_SIZE); 1084 } else { 1085 pte = (pte_t *) (paddr + PAGE_SIZE); 1086 to_free = (unsigned long *) paddr; 1087 } 1088 1089#if (L1DCACHE_SIZE > PAGE_SIZE) /* is there D$ aliasing problem */ 1090 /* Now free the other one up, adjust cache size. */ 1091 *to_free = (unsigned long) pte_quicklist[color ^ 0x1]; 1092 pte_quicklist[color ^ 0x1] = to_free; 1093 pgtable_cache_size++; 1094#endif 1095 1096 return pte; 1097 } 1098 return NULL; 1099} 1100 1101void sparc_ultra_dump_itlb(void) 1102{ 1103 int slot; 1104 1105 if (tlb_type == spitfire) { 1106 printk ("Contents of itlb: "); 1107 for (slot = 0; slot < 14; slot++) printk (" "); 1108 printk ("%2x:%016lx,%016lx\n", 1109 0, 1110 spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0)); 1111 for (slot = 1; slot < 64; slot+=3) { 1112 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1113 slot, 1114 spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot), 1115 slot+1, 1116 spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1), 1117 slot+2, 1118 spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2)); 1119 } 1120 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1121 printk ("Contents of itlb0:\n"); 1122 for (slot = 0; slot < 16; slot+=2) { 1123 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1124 slot, 1125 cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot), 1126 slot+1, 1127 cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1)); 1128 } 1129 printk ("Contents of itlb2:\n"); 1130 for (slot = 0; slot < 128; slot+=2) { 1131 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1132 slot, 1133 cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot), 1134 slot+1, 1135 cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1)); 1136 } 1137 } 1138} 1139 1140void sparc_ultra_dump_dtlb(void) 1141{ 1142 int slot; 1143 1144 if (tlb_type == spitfire) { 1145 printk ("Contents of dtlb: "); 1146 for (slot = 0; slot < 14; slot++) printk (" "); 1147 printk ("%2x:%016lx,%016lx\n", 0, 1148 spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0)); 1149 for (slot = 1; slot < 64; slot+=3) { 1150 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1151 slot, 1152 spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot), 1153 slot+1, 1154 spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1), 1155 slot+2, 1156 spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2)); 1157 } 1158 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1159 printk ("Contents of dtlb0:\n"); 1160 for (slot = 0; slot < 16; slot+=2) { 1161 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1162 slot, 1163 cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot), 1164 slot+1, 1165 cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1)); 1166 } 1167 printk ("Contents of dtlb2:\n"); 1168 for (slot = 0; slot < 512; slot+=2) { 1169 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1170 slot, 1171 cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2), 1172 slot+1, 1173 cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2)); 1174 } 1175 if (tlb_type == cheetah_plus) { 1176 printk ("Contents of dtlb3:\n"); 1177 for (slot = 0; slot < 512; slot+=2) { 1178 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n", 1179 slot, 1180 cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3), 1181 slot+1, 1182 cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3)); 1183 } 1184 } 1185 } 1186} 1187 1188extern unsigned long cmdline_memory_size; 1189 1190unsigned long __init bootmem_init(unsigned long *pages_avail) 1191{ 1192 unsigned long bootmap_size, start_pfn, end_pfn; 1193 unsigned long end_of_phys_memory = 0UL; 1194 unsigned long bootmap_pfn, bytes_avail, size; 1195 int i; 1196 1197 bytes_avail = 0UL; 1198 for (i = 0; sp_banks[i].num_bytes != 0; i++) { 1199 end_of_phys_memory = sp_banks[i].base_addr + 1200 sp_banks[i].num_bytes; 1201 bytes_avail += sp_banks[i].num_bytes; 1202 if (cmdline_memory_size) { 1203 if (bytes_avail > cmdline_memory_size) { 1204 unsigned long slack = bytes_avail - cmdline_memory_size; 1205 1206 bytes_avail -= slack; 1207 end_of_phys_memory -= slack; 1208 1209 sp_banks[i].num_bytes -= slack; 1210 if (sp_banks[i].num_bytes == 0) { 1211 sp_banks[i].base_addr = 0xdeadbeef; 1212 } else { 1213 sp_banks[i+1].num_bytes = 0; 1214 sp_banks[i+1].base_addr = 0xdeadbeef; 1215 } 1216 break; 1217 } 1218 } 1219 } 1220 1221 *pages_avail = bytes_avail >> PAGE_SHIFT; 1222 1223 /* Start with page aligned address of last symbol in kernel 1224 * image. The kernel is hard mapped below PAGE_OFFSET in a 1225 * 4MB locked TLB translation. 1226 */ 1227 start_pfn = PAGE_ALIGN((unsigned long) &_end) - 1228 ((unsigned long) KERNBASE); 1229 1230 /* Adjust up to the physical address where the kernel begins. */ 1231 start_pfn += phys_base; 1232 1233 /* Now shift down to get the real physical page frame number. */ 1234 start_pfn >>= PAGE_SHIFT; 1235 1236 bootmap_pfn = start_pfn; 1237 1238 end_pfn = end_of_phys_memory >> PAGE_SHIFT; 1239 1240#ifdef CONFIG_BLK_DEV_INITRD 1241 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */ 1242 if (sparc_ramdisk_image) { 1243 if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE) 1244 sparc_ramdisk_image -= KERNBASE; 1245 initrd_start = sparc_ramdisk_image + phys_base; 1246 initrd_end = initrd_start + sparc_ramdisk_size; 1247 if (initrd_end > end_of_phys_memory) { 1248 printk(KERN_CRIT "initrd extends beyond end of memory " 1249 "(0x%016lx > 0x%016lx)\ndisabling initrd\n", 1250 initrd_end, end_of_phys_memory); 1251 initrd_start = 0; 1252 } 1253 if (initrd_start) { 1254 if (initrd_start >= (start_pfn << PAGE_SHIFT) && 1255 initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE) 1256 bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT; 1257 } 1258 } 1259#endif 1260 /* Initialize the boot-time allocator. */ 1261 max_pfn = max_low_pfn = end_pfn; 1262 min_low_pfn = phys_base >> PAGE_SHIFT; 1263 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, phys_base>>PAGE_SHIFT, end_pfn); 1264 1265 /* Now register the available physical memory with the 1266 * allocator. 1267 */ 1268 for (i = 0; sp_banks[i].num_bytes != 0; i++) 1269 free_bootmem(sp_banks[i].base_addr, 1270 sp_banks[i].num_bytes); 1271 1272#ifdef CONFIG_BLK_DEV_INITRD 1273 if (initrd_start) { 1274 size = initrd_end - initrd_start; 1275 1276 /* Resert the initrd image area. */ 1277 reserve_bootmem(initrd_start, size); 1278 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; 1279 1280 initrd_start += PAGE_OFFSET; 1281 initrd_end += PAGE_OFFSET; 1282 } 1283#endif 1284 /* Reserve the kernel text/data/bss. */ 1285 size = (start_pfn << PAGE_SHIFT) - phys_base; 1286 reserve_bootmem(phys_base, size); 1287 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; 1288 1289 /* Reserve the bootmem map. We do not account for it 1290 * in pages_avail because we will release that memory 1291 * in free_all_bootmem. 1292 */ 1293 size = bootmap_size; 1294 reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size); 1295 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; 1296 1297 return end_pfn; 1298} 1299 1300/* paging_init() sets up the page tables */ 1301 1302extern void sun_serial_setup(void); 1303extern void cheetah_ecache_flush_init(void); 1304 1305static unsigned long last_valid_pfn; 1306 1307void __init paging_init(void) 1308{ 1309 extern pmd_t swapper_pmd_dir[1024]; 1310 extern unsigned int sparc64_vpte_patchme1[1]; 1311 extern unsigned int sparc64_vpte_patchme2[1]; 1312 unsigned long alias_base = phys_base + PAGE_OFFSET; 1313 unsigned long second_alias_page = 0; 1314 unsigned long pt, flags, end_pfn, pages_avail; 1315 unsigned long shift = alias_base - ((unsigned long)KERNBASE); 1316 unsigned long real_end; 1317 1318 set_bit(0, mmu_context_bmap); 1319 1320 real_end = (unsigned long)&_end; 1321 if ((real_end > ((unsigned long)KERNBASE + 0x400000))) 1322 bigkernel = 1; 1323#ifdef CONFIG_BLK_DEV_INITRD 1324 if (sparc_ramdisk_image) 1325 real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size)); 1326#endif 1327 1328 /* We assume physical memory starts at some 4mb multiple, 1329 * if this were not true we wouldn't boot up to this point 1330 * anyways. 1331 */ 1332 pt = phys_base | _PAGE_VALID | _PAGE_SZ4MB; 1333 pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W; 1334 __save_and_cli(flags); 1335 if (tlb_type == spitfire) { 1336 __asm__ __volatile__( 1337 " stxa %1, [%0] %3\n" 1338 " stxa %2, [%5] %4\n" 1339 " membar #Sync\n" 1340 " flush %%g6\n" 1341 " nop\n" 1342 " nop\n" 1343 " nop\n" 1344 : /* No outputs */ 1345 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt), 1346 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3) 1347 : "memory"); 1348 if (real_end >= KERNBASE + 0x340000) { 1349 second_alias_page = alias_base + 0x400000; 1350 __asm__ __volatile__( 1351 " stxa %1, [%0] %3\n" 1352 " stxa %2, [%5] %4\n" 1353 " membar #Sync\n" 1354 " flush %%g6\n" 1355 " nop\n" 1356 " nop\n" 1357 " nop\n" 1358 : /* No outputs */ 1359 : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000), 1360 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3) 1361 : "memory"); 1362 } 1363 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1364 __asm__ __volatile__( 1365 " stxa %1, [%0] %3\n" 1366 " stxa %2, [%5] %4\n" 1367 " membar #Sync\n" 1368 " flush %%g6\n" 1369 " nop\n" 1370 " nop\n" 1371 " nop\n" 1372 : /* No outputs */ 1373 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt), 1374 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3)) 1375 : "memory"); 1376 if (real_end >= KERNBASE + 0x340000) { 1377 second_alias_page = alias_base + 0x400000; 1378 __asm__ __volatile__( 1379 " stxa %1, [%0] %3\n" 1380 " stxa %2, [%5] %4\n" 1381 " membar #Sync\n" 1382 " flush %%g6\n" 1383 " nop\n" 1384 " nop\n" 1385 " nop\n" 1386 : /* No outputs */ 1387 : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000), 1388 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3)) 1389 : "memory"); 1390 } 1391 } 1392 __restore_flags(flags); 1393 1394 /* Now set kernel pgd to upper alias so physical page computations 1395 * work. 1396 */ 1397 init_mm.pgd += ((shift) / (sizeof(pgd_t))); 1398 1399 memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir)); 1400 1401 /* Now can init the kernel/bad page tables. */ 1402 pgd_set(&swapper_pg_dir[0], swapper_pmd_dir + (shift / sizeof(pgd_t))); 1403 1404 sparc64_vpte_patchme1[0] |= (pgd_val(init_mm.pgd[0]) >> 10); 1405 sparc64_vpte_patchme2[0] |= (pgd_val(init_mm.pgd[0]) & 0x3ff); 1406 flushi((long)&sparc64_vpte_patchme1[0]); 1407 1408 /* Setup bootmem... */ 1409 pages_avail = 0; 1410 last_valid_pfn = end_pfn = bootmem_init(&pages_avail); 1411 1412#ifdef CONFIG_SUN_SERIAL 1413 /* This does not logically belong here, but we need to 1414 * call it at the moment we are able to use the bootmem 1415 * allocator. 1416 */ 1417 sun_serial_setup(); 1418#endif 1419 1420 /* Inherit non-locked OBP mappings. */ 1421 inherit_prom_mappings(); 1422 1423 /* Ok, we can use our TLB miss and window trap handlers safely. 1424 * We need to do a quick peek here to see if we are on StarFire 1425 * or not, so setup_tba can setup the IRQ globals correctly (it 1426 * needs to get the hard smp processor id correctly). 1427 */ 1428 { 1429 extern void setup_tba(int); 1430 setup_tba(this_is_starfire); 1431 } 1432 1433 inherit_locked_prom_mappings(1); 1434 1435 /* We only created DTLB mapping of this stuff. */ 1436 spitfire_flush_dtlb_nucleus_page(alias_base); 1437 if (second_alias_page) 1438 spitfire_flush_dtlb_nucleus_page(second_alias_page); 1439 1440 __flush_tlb_all(); 1441 1442 { 1443 unsigned long zones_size[MAX_NR_ZONES]; 1444 unsigned long zholes_size[MAX_NR_ZONES]; 1445 unsigned long npages; 1446 int znum; 1447 1448 for (znum = 0; znum < MAX_NR_ZONES; znum++) 1449 zones_size[znum] = zholes_size[znum] = 0; 1450 1451 npages = end_pfn - (phys_base >> PAGE_SHIFT); 1452 zones_size[ZONE_DMA] = npages; 1453 zholes_size[ZONE_DMA] = npages - pages_avail; 1454 1455 free_area_init_node(0, NULL, NULL, zones_size, 1456 phys_base, zholes_size); 1457 } 1458 1459 device_scan(); 1460} 1461 1462/* Ok, it seems that the prom can allocate some more memory chunks 1463 * as a side effect of some prom calls we perform during the 1464 * boot sequence. My most likely theory is that it is from the 1465 * prom_set_traptable() call, and OBP is allocating a scratchpad 1466 * for saving client program register state etc. 1467 */ 1468void __init sort_memlist(struct linux_mlist_p1275 *thislist) 1469{ 1470 int swapi = 0; 1471 int i, mitr; 1472 unsigned long tmpaddr, tmpsize; 1473 unsigned long lowest; 1474 1475 for (i = 0; thislist[i].theres_more != 0; i++) { 1476 lowest = thislist[i].start_adr; 1477 for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++) 1478 if (thislist[mitr].start_adr < lowest) { 1479 lowest = thislist[mitr].start_adr; 1480 swapi = mitr; 1481 } 1482 if (lowest == thislist[i].start_adr) 1483 continue; 1484 tmpaddr = thislist[swapi].start_adr; 1485 tmpsize = thislist[swapi].num_bytes; 1486 for (mitr = swapi; mitr > i; mitr--) { 1487 thislist[mitr].start_adr = thislist[mitr-1].start_adr; 1488 thislist[mitr].num_bytes = thislist[mitr-1].num_bytes; 1489 } 1490 thislist[i].start_adr = tmpaddr; 1491 thislist[i].num_bytes = tmpsize; 1492 } 1493} 1494 1495void __init rescan_sp_banks(void) 1496{ 1497 struct linux_prom64_registers memlist[64]; 1498 struct linux_mlist_p1275 avail[64], *mlist; 1499 unsigned long bytes, base_paddr; 1500 int num_regs, node = prom_finddevice("/memory"); 1501 int i; 1502 1503 num_regs = prom_getproperty(node, "available", 1504 (char *) memlist, sizeof(memlist)); 1505 num_regs = (num_regs / sizeof(struct linux_prom64_registers)); 1506 for (i = 0; i < num_regs; i++) { 1507 avail[i].start_adr = memlist[i].phys_addr; 1508 avail[i].num_bytes = memlist[i].reg_size; 1509 avail[i].theres_more = &avail[i + 1]; 1510 } 1511 avail[i - 1].theres_more = NULL; 1512 sort_memlist(avail); 1513 1514 mlist = &avail[0]; 1515 i = 0; 1516 bytes = mlist->num_bytes; 1517 base_paddr = mlist->start_adr; 1518 1519 sp_banks[0].base_addr = base_paddr; 1520 sp_banks[0].num_bytes = bytes; 1521 1522 while (mlist->theres_more != NULL){ 1523 i++; 1524 mlist = mlist->theres_more; 1525 bytes = mlist->num_bytes; 1526 if (i >= SPARC_PHYS_BANKS-1) { 1527 printk ("The machine has more banks than " 1528 "this kernel can support\n" 1529 "Increase the SPARC_PHYS_BANKS " 1530 "setting (currently %d)\n", 1531 SPARC_PHYS_BANKS); 1532 i = SPARC_PHYS_BANKS-1; 1533 break; 1534 } 1535 1536 sp_banks[i].base_addr = mlist->start_adr; 1537 sp_banks[i].num_bytes = mlist->num_bytes; 1538 } 1539 1540 i++; 1541 sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL; 1542 sp_banks[i].num_bytes = 0; 1543 1544 for (i = 0; sp_banks[i].num_bytes != 0; i++) 1545 sp_banks[i].num_bytes &= PAGE_MASK; 1546} 1547 1548static void __init taint_real_pages(void) 1549{ 1550 struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS]; 1551 int i; 1552 1553 for (i = 0; i < SPARC_PHYS_BANKS; i++) { 1554 saved_sp_banks[i].base_addr = 1555 sp_banks[i].base_addr; 1556 saved_sp_banks[i].num_bytes = 1557 sp_banks[i].num_bytes; 1558 } 1559 1560 rescan_sp_banks(); 1561 1562 /* Find changes discovered in the sp_bank rescan and 1563 * reserve the lost portions in the bootmem maps. 1564 */ 1565 for (i = 0; saved_sp_banks[i].num_bytes; i++) { 1566 unsigned long old_start, old_end; 1567 1568 old_start = saved_sp_banks[i].base_addr; 1569 old_end = old_start + 1570 saved_sp_banks[i].num_bytes; 1571 while (old_start < old_end) { 1572 int n; 1573 1574 for (n = 0; sp_banks[n].num_bytes; n++) { 1575 unsigned long new_start, new_end; 1576 1577 new_start = sp_banks[n].base_addr; 1578 new_end = new_start + sp_banks[n].num_bytes; 1579 1580 if (new_start <= old_start && 1581 new_end >= (old_start + PAGE_SIZE)) { 1582 set_bit (old_start >> 22, 1583 sparc64_valid_addr_bitmap); 1584 goto do_next_page; 1585 } 1586 } 1587 reserve_bootmem(old_start, PAGE_SIZE); 1588 1589 do_next_page: 1590 old_start += PAGE_SIZE; 1591 } 1592 } 1593} 1594 1595void __init mem_init(void) 1596{ 1597 unsigned long codepages, datapages, initpages; 1598 unsigned long addr, last; 1599 int i; 1600 1601 i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6); 1602 i += 1; 1603 sparc64_valid_addr_bitmap = (unsigned long *) 1604 __alloc_bootmem(i << 3, SMP_CACHE_BYTES, 0UL); 1605 if (sparc64_valid_addr_bitmap == NULL) { 1606 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n"); 1607 prom_halt(); 1608 } 1609 memset(sparc64_valid_addr_bitmap, 0, i << 3); 1610 1611 addr = PAGE_OFFSET + phys_base; 1612 last = PAGE_ALIGN((unsigned long)&_end) - 1613 ((unsigned long) KERNBASE); 1614 last += PAGE_OFFSET + phys_base; 1615 while (addr < last) { 1616 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap); 1617 addr += PAGE_SIZE; 1618 } 1619 1620 taint_real_pages(); 1621 1622 max_mapnr = last_valid_pfn - (phys_base >> PAGE_SHIFT); 1623 high_memory = __va(last_valid_pfn << PAGE_SHIFT); 1624 1625 num_physpages = free_all_bootmem() - 1; 1626 1627 /* 1628 * Set up the zero page, mark it reserved, so that page count 1629 * is not manipulated when freeing the page from user ptes. 1630 */ 1631 mem_map_zero = _alloc_pages(GFP_KERNEL, 0); 1632 if (mem_map_zero == NULL) { 1633 prom_printf("paging_init: Cannot alloc zero page.\n"); 1634 prom_halt(); 1635 } 1636 SetPageReserved(mem_map_zero); 1637 clear_page(page_address(mem_map_zero)); 1638 1639 codepages = (((unsigned long) &etext) - ((unsigned long)&_start)); 1640 codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT; 1641 datapages = (((unsigned long) &edata) - ((unsigned long)&etext)); 1642 datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT; 1643 initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin)); 1644 initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT; 1645 1646#ifndef CONFIG_SMP 1647 { 1648 /* Put empty_pg_dir on pgd_quicklist */ 1649 extern pgd_t empty_pg_dir[1024]; 1650 unsigned long addr = (unsigned long)empty_pg_dir; 1651 unsigned long alias_base = phys_base + PAGE_OFFSET - 1652 (long)(KERNBASE); 1653 1654 memset(empty_pg_dir, 0, sizeof(empty_pg_dir)); 1655 addr += alias_base; 1656 free_pgd_fast((pgd_t *)addr); 1657 num_physpages++; 1658 } 1659#endif 1660 1661 printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n", 1662 nr_free_pages() << (PAGE_SHIFT-10), 1663 codepages << (PAGE_SHIFT-10), 1664 datapages << (PAGE_SHIFT-10), 1665 initpages << (PAGE_SHIFT-10), 1666 PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT)); 1667 1668 if (tlb_type == cheetah || tlb_type == cheetah_plus) 1669 cheetah_ecache_flush_init(); 1670} 1671 1672void free_initmem (void) 1673{ 1674 unsigned long addr, initend; 1675 1676 /* 1677 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes. 1678 */ 1679 addr = PAGE_ALIGN((unsigned long)(&__init_begin)); 1680 initend = (unsigned long)(&__init_end) & PAGE_MASK; 1681 for (; addr < initend; addr += PAGE_SIZE) { 1682 unsigned long page; 1683 struct page *p; 1684 1685 page = (addr + 1686 ((unsigned long) __va(phys_base)) - 1687 ((unsigned long) KERNBASE)); 1688 p = virt_to_page(page); 1689 1690 ClearPageReserved(p); 1691 set_page_count(p, 1); 1692 __free_page(p); 1693 num_physpages++; 1694 } 1695} 1696 1697#ifdef CONFIG_BLK_DEV_INITRD 1698void free_initrd_mem(unsigned long start, unsigned long end) 1699{ 1700 if (start < end) 1701 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10); 1702 for (; start < end; start += PAGE_SIZE) { 1703 struct page *p = virt_to_page(start); 1704 1705 ClearPageReserved(p); 1706 set_page_count(p, 1); 1707 __free_page(p); 1708 num_physpages++; 1709 } 1710} 1711#endif 1712 1713void si_meminfo(struct sysinfo *val) 1714{ 1715 val->totalram = num_physpages; 1716 val->sharedram = 0; 1717 val->freeram = nr_free_pages(); 1718 val->bufferram = atomic_read(&buffermem_pages); 1719 1720 /* These are always zero on Sparc64. */ 1721 val->totalhigh = 0; 1722 val->freehigh = 0; 1723 1724 val->mem_unit = PAGE_SIZE; 1725} 1726