1/*
2 * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License").  You may not use this file except in compliance with the
9 * License.  Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22// 45678901234567890123456789012345678901234567890123456789012345678901234567890
23
24#include <libkern/OSAtomic.h>
25
26#include <IOKit/IOLocks.h>
27#include <IOKit/IOPlatformExpert.h>
28#include <IOKit/IODeviceTreeSupport.h>
29
30#include <IOKit/IOMapper.h>
31
32extern "C" {
33extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
34}
35
36/*
37 * Register layout defines
38 */
39
40//                00000000001111111111222222222233
41//                01234567890123456789012345678901
42//                10987654321098765432109876543210
43//                33222222222211111111110000000000
44
45// U3
46
47// DARTCNTL Register format
48// DARTBASE       11111111111111111111		        ffff f    20b x << 12
49// IDARTTLB                            1	              4    1b x << 10
50// DARTEN                               1	              2    1b x <<  9
51// DARTSIZE                              111111111            1ff  9b x <<  0
52
53// DARTTAG Register format
54// DARTLPN        1111111111111	                        fff8      13b x << 19
55// DARTSET                     11111                       7e      5b x << 14
56
57
58// U4
59
60// DARTCNTL Register format
61// DARTEN         1                                     8          1b x << 31
62// IDARTTLB         1                                   2          1b x << 29
63
64// DARTBASE Register format
65// DARTBASE              111111111111111111111111       ff ffff   24b x <<  0
66
67// DARTSIZE Register format
68// DARTSIZE                     11111111111111111          1fff   17b x <<  0
69
70// DARTTAG Register format
71// DARTLPN               1111111111111                  fff8      13b x << 19
72// DARTSET                            11111                7e      5b x << 14
73
74#define U3DARTBASEMASK	 0xfffff
75#define U3DARTBASESHFT	      12
76#define U4DARTBASEMASK	0xffffff
77#define U4DARTBASESHFT	       0
78
79#define U3IDARTTLB		0x00000400	// 1 << 10
80#define U3DARTEN		0x00000200	// 1 <<  9
81#define U4DARTEN		0x80000000	// 1 << 31
82#define U4IALLTLB		0x20000000	// 1 << 29
83
84#define U3DARTSIZEMASK	   0x1ff
85#define U3DARTSIZESHFT	       0
86#define U4DARTSIZEMASK	  0x1fff
87#define U4DARTSIZESHFT	       0
88
89// #define DARTLPNMASK	  0x1fff
90// #define DARTLPNSHFT	      19
91
92// #define DARTVxMASK	     0xf
93// #define DARTVxSHFT	      14
94
95// These are with respect to the PCI address, i.e. not page number based.
96// #define DARTSETMASK         0x1f
97// #define DARTSETSHFT           14
98
99// #define DARTWAYSETMASK      0x7f
100// #define DARTWAYSETSHFT        14
101
102#define DARTCNTL	0x0000
103#define DARTBASE	0x0010
104#define DARTSIZE	0x0020
105#define DARTEXCPU3	0x0010
106#define DARTEXCPU4	0x0030
107// #define DARTTAG	0x1000
108// #define DARTTAGSZ	0x0740
109// #define DARTDATA	0x5000
110// #define DARTDATASZ	0x1ff0
111
112#define DARTTLBPAGES	     4
113
114#define frmDARTTLB(tlb) ((Uint32 *) \
115    ( (vm_address_t) fRegBase + (tlb << DARTSETSHFT) + DARTTAG) )
116
117// Convert physical ppnum_t base of DART to U3 DARTCNTL register DARTBASE bits
118#define setDARTBASE(b)	(fU3 ? (((b) & U3DARTBASEMASK) << U3DARTBASESHFT) : (((b) & U4DARTBASEMASK) << U4DARTBASESHFT))
119
120// Convert Size that DART maps in Bytes to U3 DARTCNTL register DARTSIZE bits
121#define setDARTSIZE(p)	(fU3 ? (((p) & U3DARTSIZEMASK) << U3DARTSIZESHFT) : (((p) & U4DARTSIZEMASK) << U4DARTSIZESHFT))
122
123// Set the DART Enable Bit
124#define setDARTEN() (fU3 ? U3DARTEN : U4DARTEN)
125
126// Set the DART Invalidate All Bit
127#define setIALLTLB() (fU3 ? U3IDARTTLB : U4IALLTLB)
128
129// Convert a physical ppnum_t into DARTTAG register DARTLPN bits
130// #define invalidDARTLPN(a) ( (a) & (DARTLPNMASK << DARTLPNSHFT) )
131
132// General constants about all VART/DART style Address Re-Mapping Tables
133#define kMapperPage	    (4 * 1024)
134#define kTransPerPage	    (kMapperPage / sizeof(ppnum_t))
135
136// 2Gb System table and 32Mb Regular (i.e. not system) table
137#define kSysTableMapped	    (2UL * 1024 * 1024 * 1024)
138#define kRegTableMapped	    (32UL * 1024 * 1024)
139
140#define kSysMappedPages	    (kSysTableMapped / kMapperPage)
141#define kRegMappedPages	    (kRegTableMapped / kMapperPage)
142
143#define kSysDARTSize	    (kSysMappedPages / kTransPerPage)
144#define kRegDARTSize	    (kRegMappedPages / kTransPerPage)
145
146#define kMinZoneSize	    4		// Minimum Zone size in pages
147#define kMaxNumZones	    (31 - 14)	// 31 bit mapped in 16K super pages
148
149#define super IOMapper
150
151class AppleDART : public IOMapper
152{
153    OSDeclareDefaultStructors(AppleDART);
154
155// alias the fTable variable into our mappings table
156#define fMappings	((volatile ppnum_t *) super::fTable)
157
158private:
159
160    static vm_size_t	gCacheLineSize;
161
162    UInt32		fFreeLists[kMaxNumZones];
163
164    IOLock		*fTableLock;
165    IOSimpleLock	*fInvalidateLock;
166
167    void		*fDummyPage;
168
169    IOMemoryMap		*fRegisterVMMap;
170    UInt8		*fRegBase;
171
172    volatile UInt32	*fDARTCNTLReg;
173    volatile UInt32	*fDARTBASEReg;
174    volatile UInt32	*fDARTSIZEReg;
175    volatile UInt32	*fDARTEXCPReg;
176
177//  volatile UInt32	*fRegTagBase;
178//  IOMemoryMap		*fTagVMMap;
179
180// #define fDARTTAGReg	((volatile UInt32 *) (fRegBase + DARTTAG))
181// #define fDARTDATAPReg	((volatile UInt32 *) (fRegBase + DARTDATA))
182
183    UInt32		 fDARTCNTLVal;	// Shadow value of DARTCNTL register
184    UInt32		 fNumZones;
185    UInt32		 fMapperRegionSize;
186    UInt32		 fFreeSleepers;
187    ppnum_t		 fDummyPageNumber;
188
189    bool		fU3;
190
191    // Internal functions
192    void flushMappings(volatile void *vaddr, UInt32 numMappings);
193
194    void breakUp(unsigned start, unsigned end, unsigned freeInd);
195    void invalidateDART(ppnum_t pnum, IOItemCount size);
196    void tlbInvalidate(ppnum_t pnum, IOItemCount size);
197
198    virtual void free();
199
200    virtual bool initHardware(IOService *provider);
201
202    virtual ppnum_t iovmAlloc(IOItemCount pages);
203    virtual void iovmFree(ppnum_t addr, IOItemCount pages);
204
205    virtual void iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page);
206    virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
207                            ppnum_t *pageList, IOItemCount pageCount);
208    virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
209                            upl_page_info_t *pageList, IOItemCount pageCount);
210
211    virtual addr64_t mapAddr(IOPhysicalAddress addr);
212};
213
214vm_size_t AppleDART::gCacheLineSize = 32;
215
216OSDefineMetaClassAndStructors(AppleDART, IOMapper);
217
218static inline void writeReg(volatile UInt32 *reg, UInt32 value)
219{
220    *reg = value;
221    OSSynchronizeIO();
222}
223
224static inline UInt32 readReg(volatile UInt32 *reg)
225{
226    return *reg;
227}
228
229// Remember no value can be bigger than 31 bits as the sign bit indicates
230// that this entry is valid to the hardware and that would be bad if it wasn't
231typedef struct FreeDARTEntry {
232    unsigned int
233    /* bool */	    fValid : 1,
234    /* bool */	    fInUse : 1,	// Allocated but not inserted yet
235    /* bool */		   : 5,	// Align size on nibble boundary for debugging
236    /* uint */	    fSize  : 5,
237    /* uint */	           : 2,
238    /* uint */	    fNext  :18;	// offset of FreeDARTEntry's
239    unsigned int
240    /* uint */	           :14,
241    /* uint */	    fPrev  :18;	// offset of FreeDARTEntry's
242} FreeDARTEntry;
243
244#define kInvalidInUse	0x40000000
245
246typedef struct ActiveDARTEntry {
247    unsigned int
248    /* bool */	    fValid : 1,	// Must be set to one if valid
249    /* uint */	           :12, // Don't care
250    /* uint */	    fPPNum :19;	// ppnum_t page of translation
251};
252#define kValidEntry 0x80000000
253
254#define kActivePerFree (sizeof(freeDART[0]) / sizeof(ActiveDARTEntry))
255
256bool AppleDART::initHardware(IOService *provider)
257{
258    IOPlatformDevice *nub = OSDynamicCast(IOPlatformDevice, provider);
259
260    if (!nub)
261	return false;
262
263    UInt32 dartSizePages = 0;
264
265    fIsSystem = true;
266
267    if (fIsSystem && !IOMapper::gSystem) {
268	IOLog("DART disabled\n");
269	kprintf("DART disabled\n");
270	return false;
271    }
272
273    if (IODTMatchNubWithKeys(provider, "('u3-dart')"))
274	fU3 = true;
275
276    OSNumber *sizeNum = (OSNumber *) getProperty("AppleARTSize");
277    if (sizeNum)
278	dartSizePages = sizeNum->unsigned32BitValue();
279
280    fTableLock = IOLockAlloc();
281    fInvalidateLock = IOSimpleLockAlloc();
282
283    if (!fTableLock || !fInvalidateLock)
284	return false;
285
286    if (fIsSystem) {
287	int bootArg;
288
289	// @@@ gvdl: Must be a guppy, there has to be a better way.
290	gCacheLineSize = 128;
291	if (PE_parse_boot_arg("artsize", &bootArg ))
292	    dartSizePages = bootArg;
293	if (!dartSizePages)
294	    dartSizePages = kSysDARTSize;
295    }
296    else if (!dartSizePages)
297	dartSizePages = kRegDARTSize;
298
299    if (!allocTable((dartSizePages + 1) * kMapperPage))
300	return false;
301
302    fRegisterVMMap = nub->mapDeviceMemoryWithIndex(0);
303    if (!fRegisterVMMap)
304	return false;
305    fRegBase = (UInt8 *) fRegisterVMMap->getVirtualAddress();
306
307    if (fU3) {
308	fDARTCNTLReg = (volatile UInt32 *)(fRegBase + DARTCNTL);
309	fDARTBASEReg = (volatile UInt32 *)(fRegBase + DARTCNTL);
310	fDARTSIZEReg = (volatile UInt32 *)(fRegBase + DARTCNTL);
311	fDARTEXCPReg = (volatile UInt32 *)(fRegBase + DARTEXCPU3);
312    } else {
313	fDARTCNTLReg = (volatile UInt32 *)(fRegBase + DARTCNTL);
314	fDARTBASEReg = (volatile UInt32 *)(fRegBase + DARTBASE);
315	fDARTSIZEReg = (volatile UInt32 *)(fRegBase + DARTSIZE);
316	fDARTEXCPReg = (volatile UInt32 *)(fRegBase + DARTEXCPU4);
317    }
318
319    UInt32 canMapPages = dartSizePages * kTransPerPage;
320    fMapperRegionSize = canMapPages;
321    for (fNumZones = 0; canMapPages; fNumZones++)
322	canMapPages >>= 1;
323    fNumZones -= 3; // correct for overshoot and minumum 16K pages allocation
324
325    fDARTCNTLVal = readReg(fDARTSIZEReg) | setDARTSIZE(dartSizePages);
326    writeReg(fDARTSIZEReg, fDARTCNTLVal);
327
328    fDARTCNTLVal = readReg(fDARTBASEReg) | setDARTBASE(fTablePhys);
329    writeReg(fDARTBASEReg, fDARTCNTLVal);
330
331    fDARTCNTLVal |= setDARTEN(); // Enable the DART
332    writeReg(fDARTCNTLReg, fDARTCNTLVal);
333
334    fDARTCNTLVal |= setIALLTLB();	// Assert Invalidate TLB bit in shadow
335    invalidateDART(0, dartSizePages * kTransPerPage);
336
337    breakUp(0, fNumZones, 0);
338    *(ppnum_t *) fTable = kInvalidInUse;
339
340    fDummyPage = IOMallocAligned(0x1000, 0x1000);
341    fDummyPageNumber =
342	pmap_find_phys(kernel_pmap, (addr64_t) (uintptr_t) fDummyPage);
343
344    IOLog("DART enabled\n");
345    kprintf("DART enabled\n");
346
347    return true;
348}
349
350void AppleDART::free()
351{
352    if (fDummyPage) {
353	IOFreeAligned(fDummyPage, 0x1000);
354	fDummyPage = 0;
355	fDummyPageNumber = 0;
356    }
357
358    if (fRegisterVMMap) {
359	fRegBase = 0;
360	fRegisterVMMap->release();
361	fRegisterVMMap = 0;
362    }
363
364#if 0
365    if (fTagVMMap) {
366	fRegTagBase = 0;
367	fTagVMMap->release();
368	fTagVMMap = 0;
369    }
370#endif
371
372    if (fTableLock) {
373	IOLockFree(fTableLock);
374	fTableLock = 0;
375    }
376    if (fInvalidateLock) {
377	IOSimpleLockFree(fInvalidateLock);
378	fInvalidateLock = 0;
379    }
380
381    super::free();
382}
383
384// Must be called while locked
385void AppleDART::breakUp(unsigned start, unsigned end, unsigned freeInd)
386{
387    unsigned int zoneSize;
388    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
389
390    do {
391	// Need to break up bigger blocks of memory till we get one in our
392	// desired zone.
393	end--;
394	zoneSize = (kMinZoneSize/2 << end);
395	ppnum_t tail = freeInd + zoneSize;
396
397	// By definition free lists must be empty
398	fFreeLists[end] = tail;
399	freeDART[tail].fSize = end;
400	freeDART[tail].fNext = freeDART[tail].fPrev = 0;
401    } while (end != start);
402    freeDART[freeInd].fSize = end;
403}
404
405// Zero is never a valid page to return
406ppnum_t AppleDART::iovmAlloc(IOItemCount pages)
407{
408    unsigned int zone, zoneSize, z, cnt;
409    ppnum_t next, ret = 0;
410    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
411
412    // Force an extra page on every allocation
413    pages += 1;
414
415    // Can't alloc anything of less than minumum
416    if (pages < kMinZoneSize)
417	pages = kMinZoneSize;
418
419    // Can't alloc anything bigger than 1/2 table
420    if (pages >= fMapperRegionSize/2)
421    {
422	panic("iovmAlloc");
423	return 0;
424    }
425
426    // Find the appropriate zone for this allocation
427    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
428	zoneSize <<= 1;
429
430    {
431	IOLockLock(fTableLock);
432
433	for (;;) {
434	    for (z = zone; z < fNumZones; z++) {
435		if ( (ret = fFreeLists[z]) )
436		    break;
437	    }
438	    if (ret)
439		break;
440
441	    fFreeSleepers++;
442	    IOLockSleep(fTableLock, fFreeLists, THREAD_UNINT);
443	    fFreeSleepers--;
444	}
445
446	// If we didn't find a entry in our size then break up the free block
447	// that we did find.
448	if (zone != z)
449	    breakUp(zone, z, ret);
450
451	freeDART[ret].fInUse = true;	// Mark entry as In Use
452	next = freeDART[ret].fNext;
453	fFreeLists[z] = next;
454	if (next)
455	    freeDART[next].fPrev = 0;
456
457	IOLockUnlock(fTableLock);
458    }
459
460    // ret is free list offset not page offset;
461    ret *= kActivePerFree;
462
463    ppnum_t pageEntry = fDummyPageNumber | kValidEntry;
464    for (cnt = 0; cnt < pages; cnt++) {
465	volatile ppnum_t *activeDART = &fMappings[ret + cnt];
466	*activeDART = pageEntry;
467    }
468
469    return ret;
470}
471
472void AppleDART::tlbInvalidate(ppnum_t pnum, IOItemCount size)
473{
474    IOSimpleLockLock(fInvalidateLock);
475
476    writeReg(fDARTCNTLReg, fDARTCNTLVal); // Invalidate the tlb
477    if (readReg(fDARTCNTLReg) & setIALLTLB()) {
478	AbsoluteTime now, expirationTime;
479	bool	     ok;
480	UInt32       loops = 0;
481
482	do {
483	    clock_interval_to_deadline(100, kNanosecondScale, &expirationTime);
484	    do {
485		ok = !(readReg(fDARTCNTLReg) & setIALLTLB());
486		if (ok)
487		    break;
488		clock_get_uptime(&now);
489	    } while (CMP_ABSOLUTETIME(&now, &expirationTime) < 0);
490	    if (ok)
491		break;
492
493	    // clear and reset IDARTTLB
494	    writeReg(fDARTCNTLReg, fDARTCNTLVal & ~setIALLTLB());
495	    writeReg(fDARTCNTLReg, fDARTCNTLVal);
496	} while (loops < 10);
497	if (!ok)
498	    panic("AppleDART IDARTTLB");
499    }
500
501    IOSimpleLockUnlock(fInvalidateLock);
502}
503
504void AppleDART::invalidateDART(ppnum_t pnum, IOItemCount size)
505{
506    // Clear out all of those valid bits in the dart table
507    if (size >= (2 * gCacheLineSize))
508	bzero((void *) &fMappings[pnum], size * sizeof(fMappings[0]));
509    else
510    {	// Arbitrary break even point
511	for (vm_size_t i = 0; i < size; i++)
512	    fMappings[pnum+i] = 0;
513    }
514
515    // Flush changes out to the U3
516    flushMappings(&fMappings[pnum], size);
517
518    tlbInvalidate(pnum, size);
519}
520
521void AppleDART::iovmFree(ppnum_t addr, IOItemCount pages)
522{
523    unsigned int zone, zoneSize, z;
524    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
525
526    // Force an extra page on every allocation
527    pages += 1;
528
529    // Can't free anything of less than minumum
530    if (pages < kMinZoneSize)
531	pages = kMinZoneSize;
532
533    // Can't free anything bigger than 1/2 table
534    if (pages >= fMapperRegionSize/2)
535	return;
536
537    // Find the appropriate zone for this allocation
538    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
539	zoneSize <<= 1;
540
541    // Grab lock that protects the dart
542    IOLockLock(fTableLock);
543
544    invalidateDART(addr, pages);
545
546    addr /= kActivePerFree;
547
548    // We are freeing a block, check to see if pairs are available for
549    // coalescing.  We will walk up the entire chain if we can.
550    for (z = zone; z < fNumZones; z++) {
551	ppnum_t pair = addr ^ (kMinZoneSize/2 << z);	// Find pair address
552	if (freeDART[pair].fValid || freeDART[pair].fInUse || (freeDART[pair].fSize != z))
553	    break;
554
555	// The paired alloc entry is free if we are here
556	ppnum_t next = freeDART[pair].fNext;
557	ppnum_t prev = freeDART[pair].fPrev;
558
559	// Remove the pair from its freeList
560	if (prev)
561	    freeDART[prev].fNext = next;
562	else
563	    fFreeLists[z] = next;
564
565	if (next)
566	    freeDART[next].fPrev = prev;
567
568	// Sort the addr and the pair
569	if (addr > pair)
570	    addr = pair;
571    }
572
573    // Add the allocation entry into it's free list and re-init it
574    freeDART[addr].fSize = z;
575    freeDART[addr].fNext = fFreeLists[z];
576    if (fFreeLists[z])
577	freeDART[fFreeLists[z]].fPrev = addr;
578    freeDART[addr].fPrev = 0;
579    fFreeLists[z] = addr;
580
581    if (fFreeSleepers)
582	IOLockWakeup(fTableLock, fFreeLists, /* oneThread */ false);
583
584    IOLockUnlock(fTableLock);
585}
586
587addr64_t AppleDART::mapAddr(IOPhysicalAddress addr)
588{
589    if (addr >= ptoa_32(fMapperRegionSize))
590    {
591	return (addr64_t) addr;	// Not mapped by us anyway
592    }
593    else
594    {
595	ppnum_t *activeDART = (ppnum_t *) fTable;
596	UInt offset = addr & PAGE_MASK;
597
598	ppnum_t mappedPage = activeDART[atop_32(addr)];
599	if (mappedPage &  kValidEntry) {
600	    mappedPage ^= kValidEntry;	// Clear validity bit
601	    return ptoa_64(mappedPage) | offset;
602	}
603	panic("%s::mapAddr(0x%08lx) not mapped for I/O\n", getName(), addr);
604	return 0;
605    }
606}
607
608void AppleDART::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page)
609{
610    addr += offset;	// Add the offset page to the base address
611
612    volatile ppnum_t *activeDART = &fMappings[addr];
613    *activeDART = page | kValidEntry;
614
615    // Flush changes out to the U3
616    flushMappings(activeDART, 1);
617
618    tlbInvalidate(addr, 1);
619}
620
621void AppleDART::iovmInsert(ppnum_t addr, IOItemCount offset,
622	ppnum_t *pageList, IOItemCount pageCount)
623{
624    addr += offset;	// Add the offset page to the base address
625
626    IOItemCount i;
627    volatile ppnum_t *activeDART = &fMappings[addr];
628
629    for (i = 0; i < pageCount; i++)
630	activeDART[i] = pageList[i] | kValidEntry;
631
632    // Flush changes out to the U3
633    flushMappings(activeDART, pageCount);
634
635    tlbInvalidate(addr, pageCount);
636}
637
638void AppleDART::iovmInsert(ppnum_t addr, IOItemCount offset,
639	upl_page_info_t *pageList, IOItemCount pageCount)
640{
641    addr += offset;	// Add the offset page to the base address
642
643    IOItemCount i;
644    volatile ppnum_t *activeDART = &fMappings[addr];
645
646    for (i = 0; i < pageCount; i++)
647	activeDART[i] = pageList[i].phys_addr | kValidEntry;
648
649    // Flush changes out to the U3
650    flushMappings(activeDART, pageCount);
651
652    tlbInvalidate(addr, pageCount);
653}
654
655static inline void __sync(void) { __asm__ ("sync"); }
656
657static inline void __isync(void) { __asm__ ("isync"); }
658
659static inline void __dcbf(vm_address_t base, unsigned long offset)
660{
661    __asm__ ("dcbf %0, %1"
662	    :
663	    : "r" (base), "r" (offset)
664	    : "r0");
665}
666
667static inline void __dcbst(vm_address_t base, unsigned long offset)
668{
669    __asm__ ("dcbst %0, %1"
670	    :
671	    : "r" (base), "r" (offset)
672	    : "r0");
673}
674
675static inline unsigned long __lwzx(vm_address_t base, unsigned long offset)
676{
677    unsigned long result;
678
679    __asm__ volatile("lwzx %0, %1, %2"
680	    : "=r" (result)
681	    : "r" (base), "r" (offset)
682	    : "r0");
683    return result;
684
685    /* return *((unsigned long *) ((unsigned char *) base + offset)); */
686}
687
688// Note to workaround an issue where the memory controller does read-ahead
689// of unmapped memory it is necessary to flush one more mapping than
690// requested by the actual call, see len initialisation below.
691void AppleDART::flushMappings(volatile void *vaddr, UInt32 numMappings)
692{
693    SInt32 csize = gCacheLineSize;
694    vm_address_t arithAddr = (vm_address_t) vaddr;
695    vm_address_t vaddr_cache_aligned = arithAddr & ~(csize-1);
696    UInt32 len = (numMappings + 1) * sizeof(fMappings[0]);	// add one
697    SInt c, end = ((SInt)((arithAddr & (csize-1)) + len)) - csize;
698
699    for (c = 0; c < end; c += csize)
700	__dcbf(vaddr_cache_aligned, c);
701
702    __sync();
703    __isync();
704    __dcbf(vaddr_cache_aligned, c);
705    __sync();
706    __isync();
707    __lwzx(vaddr_cache_aligned, c);
708    __isync();
709}
710
711