1/*
2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <pexpert/pexpert.h>
30#include <IOKit/IOWorkLoop.h>
31#include <IOKit/IOEventSource.h>
32#include <IOKit/IOInterruptEventSource.h>
33#include <IOKit/IOCommandGate.h>
34#include <IOKit/IOTimeStamp.h>
35#include <IOKit/IOKitDebug.h>
36#include <libkern/OSDebug.h>
37#include <kern/thread.h>
38
39#define super OSObject
40
41OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
42
43// Block of unused functions intended for future use
44#if __LP64__
45OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
46OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
47OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
48#else
49OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
50OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
51OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
52#endif
53OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
54OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
55OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
56OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
57OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
58
59enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
60static inline void SETP(void *addr, unsigned int flag)
61    { unsigned char *num = (unsigned char *) addr; *num |= flag; }
62static inline void CLRP(void *addr, unsigned int flag)
63    { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
64static inline bool ISSETP(void *addr, unsigned int flag)
65    { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
66
67#define fFlags loopRestart
68
69#define passiveEventChain	reserved->passiveEventChain
70
71#if IOKITSTATS
72
73#define IOStatisticsRegisterCounter() \
74do { \
75	reserved->counter = IOStatistics::registerWorkLoop(this); \
76} while(0)
77
78#define IOStatisticsUnregisterCounter() \
79do { \
80	if (reserved) \
81		IOStatistics::unregisterWorkLoop(reserved->counter); \
82} while(0)
83
84#define IOStatisticsOpenGate() \
85do { \
86	IOStatistics::countWorkLoopOpenGate(reserved->counter); \
87} while(0)
88
89#define IOStatisticsCloseGate() \
90do { \
91	IOStatistics::countWorkLoopCloseGate(reserved->counter); \
92} while(0)
93
94#define IOStatisticsAttachEventSource() \
95do { \
96	IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
97} while(0)
98
99#define IOStatisticsDetachEventSource() \
100do { \
101	IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
102} while(0)
103
104#else
105
106#define IOStatisticsRegisterCounter()
107#define IOStatisticsUnregisterCounter()
108#define IOStatisticsOpenGate()
109#define IOStatisticsCloseGate()
110#define IOStatisticsAttachEventSource()
111#define IOStatisticsDetachEventSource()
112
113#endif /* IOKITSTATS */
114
115bool IOWorkLoop::init()
116{
117    // The super init and gateLock allocation MUST be done first.
118    if ( !super::init() )
119        return false;
120
121	// Allocate our ExpansionData if it hasn't been allocated already.
122	if ( !reserved )
123	{
124		reserved = IONew(ExpansionData,1);
125		if ( !reserved )
126			return false;
127
128		bzero(reserved,sizeof(ExpansionData));
129	}
130
131#if DEBUG
132	OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
133#endif
134
135    if ( gateLock == NULL ) {
136        if ( !( gateLock = IORecursiveLockAlloc()) )
137            return false;
138    }
139
140    if ( workToDoLock == NULL ) {
141        if ( !(workToDoLock = IOSimpleLockAlloc()) )
142            return false;
143        IOSimpleLockInit(workToDoLock);
144        workToDo = false;
145    }
146
147    if (!reserved) {
148        reserved = IONew(ExpansionData, 1);
149        reserved->options = 0;
150    }
151
152    IOStatisticsRegisterCounter();
153
154    if ( controlG == NULL ) {
155        controlG = IOCommandGate::commandGate(
156            this,
157            OSMemberFunctionCast(
158                IOCommandGate::Action,
159                this,
160                &IOWorkLoop::_maintRequest));
161
162        if ( !controlG )
163            return false;
164        // Point the controlGate at the workLoop.  Usually addEventSource
165        // does this automatically.  The problem is in this case addEventSource
166        // uses the control gate and it has to be bootstrapped.
167        controlG->setWorkLoop(this);
168        if (addEventSource(controlG) != kIOReturnSuccess)
169            return false;
170    }
171
172    if ( workThread == NULL ) {
173        thread_continue_t cptr = OSMemberFunctionCast(
174            thread_continue_t,
175            this,
176            &IOWorkLoop::threadMain);
177        if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
178            return false;
179    }
180
181    (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
182    return true;
183}
184
185IOWorkLoop *
186IOWorkLoop::workLoop()
187{
188    return IOWorkLoop::workLoopWithOptions(0);
189}
190
191IOWorkLoop *
192IOWorkLoop::workLoopWithOptions(IOOptionBits options)
193{
194	IOWorkLoop *me = new IOWorkLoop;
195
196	if (me && options) {
197		me->reserved = IONew(ExpansionData,1);
198		if (!me->reserved) {
199			me->release();
200			return 0;
201		}
202		bzero(me->reserved,sizeof(ExpansionData));
203		me->reserved->options = options;
204	}
205
206	if (me && !me->init()) {
207		me->release();
208		return 0;
209	}
210
211	return me;
212}
213
214// Free is called twice:
215// First when the atomic retainCount transitions from 1 -> 0
216// Secondly when the work loop itself is commiting hari kari
217// Hence the each leg of the free must be single threaded.
218void IOWorkLoop::free()
219{
220    if (workThread) {
221	IOInterruptState is;
222
223	// If we are here then we must be trying to shut down this work loop
224	// in this case disable all of the event source, mark the loop
225	// as terminating and wakeup the work thread itself and return
226	// Note: we hold the gate across the entire operation mainly for the
227	// benefit of our event sources so we can disable them cleanly.
228	closeGate();
229
230	disableAllEventSources();
231
232        is = IOSimpleLockLockDisableInterrupt(workToDoLock);
233	SETP(&fFlags, kLoopTerminate);
234        thread_wakeup_one((void *) &workToDo);
235        IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
236
237	openGate();
238    }
239    else /* !workThread */ {
240        IOEventSource *event, *next;
241
242        for (event = eventChain; event; event = next) {
243            next = event->getNext();
244            event->setWorkLoop(0);
245            event->setNext(0);
246            event->release();
247        }
248        eventChain = 0;
249
250        for (event = passiveEventChain; event; event = next) {
251            next = event->getNext();
252            event->setWorkLoop(0);
253            event->setNext(0);
254            event->release();
255        }
256        passiveEventChain = 0;
257
258	// Either we have a partial initialization to clean up
259	// or the workThread itself is performing hari-kari.
260	// Either way clean up all of our resources and return.
261
262	if (controlG) {
263	    controlG->release();
264	    controlG = 0;
265	}
266
267	if (workToDoLock) {
268	    IOSimpleLockFree(workToDoLock);
269	    workToDoLock = 0;
270	}
271
272	if (gateLock) {
273	    IORecursiveLockFree(gateLock);
274	    gateLock = 0;
275	}
276
277	IOStatisticsUnregisterCounter();
278
279	if (reserved) {
280	    IODelete(reserved, ExpansionData, 1);
281	    reserved = 0;
282	}
283
284	super::free();
285    }
286}
287
288IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
289{
290    return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
291}
292
293IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
294{
295    return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
296}
297
298void IOWorkLoop::enableAllEventSources() const
299{
300    IOEventSource *event;
301
302    for (event = eventChain; event; event = event->getNext())
303        event->enable();
304
305    for (event = passiveEventChain; event; event = event->getNext())
306        event->enable();
307}
308
309void IOWorkLoop::disableAllEventSources() const
310{
311    IOEventSource *event;
312
313    for (event = eventChain; event; event = event->getNext())
314		event->disable();
315
316	/* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
317    for (event = passiveEventChain; event; event = event->getNext())
318        if (event != controlG)	// Don't disable the control gate
319            event->disable();
320}
321
322void IOWorkLoop::enableAllInterrupts() const
323{
324    IOEventSource *event;
325
326    for (event = eventChain; event; event = event->getNext())
327        if (OSDynamicCast(IOInterruptEventSource, event))
328            event->enable();
329}
330
331void IOWorkLoop::disableAllInterrupts() const
332{
333    IOEventSource *event;
334
335    for (event = eventChain; event; event = event->getNext())
336        if (OSDynamicCast(IOInterruptEventSource, event))
337            event->disable();
338}
339
340
341/* virtual */ bool IOWorkLoop::runEventSources()
342{
343    bool res = false;
344    bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
345    bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
346
347    closeGate();
348    if (ISSETP(&fFlags, kLoopTerminate))
349		goto abort;
350
351    if (traceWL)
352    	IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
353
354    bool more;
355    do {
356		CLRP(&fFlags, kLoopRestart);
357		more = false;
358		IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
359		workToDo = false;
360		IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
361		/* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
362		for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
363
364			if (traceES)
365				IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
366
367			more |= evnt->checkForWork();
368
369			if (traceES)
370				IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
371
372			if (ISSETP(&fFlags, kLoopTerminate))
373				goto abort;
374			else if (fFlags & kLoopRestart) {
375				more = true;
376				break;
377			}
378		}
379    } while (more);
380
381    res = true;
382
383    if (traceWL)
384    	IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
385
386abort:
387    openGate();
388    return res;
389}
390
391/* virtual */ void IOWorkLoop::threadMain()
392{
393restartThread:
394    do {
395	if ( !runEventSources() )
396	    goto exitThread;
397
398	IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
399        if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
400	    assert_wait((void *) &workToDo, false);
401	    IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
402	    thread_continue_t cptr = NULL;
403	    if (!reserved || !(kPreciousStack & reserved->options))
404		cptr = OSMemberFunctionCast(
405			thread_continue_t, this, &IOWorkLoop::threadMain);
406	    thread_block_parameter(cptr, this);
407	    goto restartThread;
408	    /* NOTREACHED */
409	}
410
411	// At this point we either have work to do or we need
412	// to commit suicide.  But no matter
413	// Clear the simple lock and retore the interrupt state
414	IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
415    } while(workToDo);
416
417exitThread:
418	thread_t thread = workThread;
419    workThread = 0;	// Say we don't have a loop and free ourselves
420    free();
421
422	thread_deallocate(thread);
423    (void) thread_terminate(thread);
424}
425
426IOThread IOWorkLoop::getThread() const
427{
428    return workThread;
429}
430
431bool IOWorkLoop::onThread() const
432{
433    return (IOThreadSelf() == workThread);
434}
435
436bool IOWorkLoop::inGate() const
437{
438    return IORecursiveLockHaveLock(gateLock);
439}
440
441// Internal APIs used by event sources to control the thread
442void IOWorkLoop::signalWorkAvailable()
443{
444    if (workToDoLock) {
445        IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
446        workToDo = true;
447        thread_wakeup_one((void *) &workToDo);
448        IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
449    }
450}
451
452void IOWorkLoop::openGate()
453{
454    IOStatisticsOpenGate();
455    IORecursiveLockUnlock(gateLock);
456}
457
458void IOWorkLoop::closeGate()
459{
460    IORecursiveLockLock(gateLock);
461    IOStatisticsCloseGate();
462}
463
464bool IOWorkLoop::tryCloseGate()
465{
466    bool res = (IORecursiveLockTryLock(gateLock) != 0);
467    if (res) {
468        IOStatisticsCloseGate();
469    }
470    return res;
471}
472
473int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
474{
475    int res;
476    IOStatisticsOpenGate();
477    res = IORecursiveLockSleep(gateLock, event, interuptibleType);
478    IOStatisticsCloseGate();
479    return res;
480}
481
482int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
483{
484    int res;
485    IOStatisticsOpenGate();
486    res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
487    IOStatisticsCloseGate();
488    return res;
489}
490
491void IOWorkLoop::wakeupGate(void *event, bool oneThread)
492{
493    IORecursiveLockWakeup(gateLock, event, oneThread);
494}
495
496IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
497                                  void *arg0, void *arg1,
498                                  void *arg2, void *arg3)
499{
500    IOReturn res;
501
502    // closeGate is recursive so don't worry if we already hold the lock.
503    closeGate();
504    res = (*inAction)(target, arg0, arg1, arg2, arg3);
505    openGate();
506
507    return res;
508}
509
510IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
511{
512    maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
513    IOEventSource *inEvent = (IOEventSource *) inD;
514    IOReturn res = kIOReturnSuccess;
515
516    switch (command)
517    {
518    case mAddEvent:
519        if (!inEvent->getWorkLoop()) {
520            SETP(&fFlags, kLoopRestart);
521
522            inEvent->retain();
523            inEvent->setWorkLoop(this);
524            inEvent->setNext(0);
525
526    		/* Check if this is a passive or active event source being added */
527    		if (eventSourcePerformsWork(inEvent)) {
528
529	            if (!eventChain)
530    	            eventChain = inEvent;
531        	    else {
532            	    IOEventSource *event, *next;
533
534                	for (event = eventChain; (next = event->getNext()); event = next)
535                    	;
536                	event->setNext(inEvent);
537
538            	}
539
540            }
541            else {
542
543	            if (!passiveEventChain)
544    	            passiveEventChain = inEvent;
545        	    else {
546            	    IOEventSource *event, *next;
547
548                	for (event = passiveEventChain; (next = event->getNext()); event = next)
549                    	;
550                	event->setNext(inEvent);
551
552            	}
553
554            }
555            IOStatisticsAttachEventSource();
556        }
557        break;
558
559    case mRemoveEvent:
560        if (inEvent->getWorkLoop()) {
561        	IOStatisticsDetachEventSource();
562
563        	if (eventSourcePerformsWork(inEvent)) {
564				if (eventChain == inEvent)
565					eventChain = inEvent->getNext();
566				else {
567					IOEventSource *event, *next;
568
569					event = eventChain;
570					while ((next = event->getNext()) && next != inEvent)
571						event = next;
572
573					if (!next) {
574						res = kIOReturnBadArgument;
575						break;
576					}
577					event->setNext(inEvent->getNext());
578				}
579    		}
580    		else {
581				if (passiveEventChain == inEvent)
582					passiveEventChain = inEvent->getNext();
583				else {
584					IOEventSource *event, *next;
585
586					event = passiveEventChain;
587					while ((next = event->getNext()) && next != inEvent)
588						event = next;
589
590					if (!next) {
591						res = kIOReturnBadArgument;
592						break;
593					}
594					event->setNext(inEvent->getNext());
595				}
596    		}
597
598            inEvent->setWorkLoop(0);
599            inEvent->setNext(0);
600            inEvent->release();
601            SETP(&fFlags, kLoopRestart);
602        }
603        break;
604
605    default:
606        return kIOReturnUnsupported;
607    }
608
609    return res;
610}
611
612bool
613IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
614{
615	bool	result = true;
616
617	/*
618	 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
619	 * The assumption is that if you override checkForWork(), you need to be
620	 * active and not passive.
621	 *
622	 * We picked a known quantity controlG that does not override
623	 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
624	 * the workloop to which this event source is getting attached.
625	 *
626	 * We do a pointer comparison on the offset in the vtable for inNewEvent against
627	 * the offset in the vtable for inReferenceEvent. This works because
628	 * IOCommandGate's slot for checkForWork() has the address of
629	 * IOEventSource::checkForWork() in it.
630	 *
631	 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
632	 * checkForWork() here. We're just testing to see if it's the same or not.
633	 *
634	 */
635	if (controlG) {
636		void *	ptr1;
637		void *	ptr2;
638
639		ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
640		ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
641
642		if (ptr1 == ptr2)
643			result = false;
644	}
645
646    return result;
647}
648