1/* 2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <pexpert/pexpert.h> 30#include <IOKit/IOWorkLoop.h> 31#include <IOKit/IOEventSource.h> 32#include <IOKit/IOInterruptEventSource.h> 33#include <IOKit/IOCommandGate.h> 34#include <IOKit/IOTimeStamp.h> 35#include <IOKit/IOKitDebug.h> 36#include <libkern/OSDebug.h> 37#include <kern/thread.h> 38 39#define super OSObject 40 41OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); 42 43// Block of unused functions intended for future use 44#if __LP64__ 45OSMetaClassDefineReservedUnused(IOWorkLoop, 0); 46OSMetaClassDefineReservedUnused(IOWorkLoop, 1); 47OSMetaClassDefineReservedUnused(IOWorkLoop, 2); 48#else 49OSMetaClassDefineReservedUsed(IOWorkLoop, 0); 50OSMetaClassDefineReservedUsed(IOWorkLoop, 1); 51OSMetaClassDefineReservedUsed(IOWorkLoop, 2); 52#endif 53OSMetaClassDefineReservedUnused(IOWorkLoop, 3); 54OSMetaClassDefineReservedUnused(IOWorkLoop, 4); 55OSMetaClassDefineReservedUnused(IOWorkLoop, 5); 56OSMetaClassDefineReservedUnused(IOWorkLoop, 6); 57OSMetaClassDefineReservedUnused(IOWorkLoop, 7); 58 59enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; 60static inline void SETP(void *addr, unsigned int flag) 61 { unsigned char *num = (unsigned char *) addr; *num |= flag; } 62static inline void CLRP(void *addr, unsigned int flag) 63 { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } 64static inline bool ISSETP(void *addr, unsigned int flag) 65 { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } 66 67#define fFlags loopRestart 68 69#define passiveEventChain reserved->passiveEventChain 70 71#if IOKITSTATS 72 73#define IOStatisticsRegisterCounter() \ 74do { \ 75 reserved->counter = IOStatistics::registerWorkLoop(this); \ 76} while(0) 77 78#define IOStatisticsUnregisterCounter() \ 79do { \ 80 if (reserved) \ 81 IOStatistics::unregisterWorkLoop(reserved->counter); \ 82} while(0) 83 84#define IOStatisticsOpenGate() \ 85do { \ 86 IOStatistics::countWorkLoopOpenGate(reserved->counter); \ 87} while(0) 88 89#define IOStatisticsCloseGate() \ 90do { \ 91 IOStatistics::countWorkLoopCloseGate(reserved->counter); \ 92} while(0) 93 94#define IOStatisticsAttachEventSource() \ 95do { \ 96 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 97} while(0) 98 99#define IOStatisticsDetachEventSource() \ 100do { \ 101 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 102} while(0) 103 104#else 105 106#define IOStatisticsRegisterCounter() 107#define IOStatisticsUnregisterCounter() 108#define IOStatisticsOpenGate() 109#define IOStatisticsCloseGate() 110#define IOStatisticsAttachEventSource() 111#define IOStatisticsDetachEventSource() 112 113#endif /* IOKITSTATS */ 114 115bool IOWorkLoop::init() 116{ 117 // The super init and gateLock allocation MUST be done first. 118 if ( !super::init() ) 119 return false; 120 121 // Allocate our ExpansionData if it hasn't been allocated already. 122 if ( !reserved ) 123 { 124 reserved = IONew(ExpansionData,1); 125 if ( !reserved ) 126 return false; 127 128 bzero(reserved,sizeof(ExpansionData)); 129 } 130 131#if DEBUG 132 OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); 133#endif 134 135 if ( gateLock == NULL ) { 136 if ( !( gateLock = IORecursiveLockAlloc()) ) 137 return false; 138 } 139 140 if ( workToDoLock == NULL ) { 141 if ( !(workToDoLock = IOSimpleLockAlloc()) ) 142 return false; 143 IOSimpleLockInit(workToDoLock); 144 workToDo = false; 145 } 146 147 if (!reserved) { 148 reserved = IONew(ExpansionData, 1); 149 reserved->options = 0; 150 } 151 152 IOStatisticsRegisterCounter(); 153 154 if ( controlG == NULL ) { 155 controlG = IOCommandGate::commandGate( 156 this, 157 OSMemberFunctionCast( 158 IOCommandGate::Action, 159 this, 160 &IOWorkLoop::_maintRequest)); 161 162 if ( !controlG ) 163 return false; 164 // Point the controlGate at the workLoop. Usually addEventSource 165 // does this automatically. The problem is in this case addEventSource 166 // uses the control gate and it has to be bootstrapped. 167 controlG->setWorkLoop(this); 168 if (addEventSource(controlG) != kIOReturnSuccess) 169 return false; 170 } 171 172 if ( workThread == NULL ) { 173 thread_continue_t cptr = OSMemberFunctionCast( 174 thread_continue_t, 175 this, 176 &IOWorkLoop::threadMain); 177 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) 178 return false; 179 } 180 181 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); 182 return true; 183} 184 185IOWorkLoop * 186IOWorkLoop::workLoop() 187{ 188 return IOWorkLoop::workLoopWithOptions(0); 189} 190 191IOWorkLoop * 192IOWorkLoop::workLoopWithOptions(IOOptionBits options) 193{ 194 IOWorkLoop *me = new IOWorkLoop; 195 196 if (me && options) { 197 me->reserved = IONew(ExpansionData,1); 198 if (!me->reserved) { 199 me->release(); 200 return 0; 201 } 202 bzero(me->reserved,sizeof(ExpansionData)); 203 me->reserved->options = options; 204 } 205 206 if (me && !me->init()) { 207 me->release(); 208 return 0; 209 } 210 211 return me; 212} 213 214// Free is called twice: 215// First when the atomic retainCount transitions from 1 -> 0 216// Secondly when the work loop itself is commiting hari kari 217// Hence the each leg of the free must be single threaded. 218void IOWorkLoop::free() 219{ 220 if (workThread) { 221 IOInterruptState is; 222 223 // If we are here then we must be trying to shut down this work loop 224 // in this case disable all of the event source, mark the loop 225 // as terminating and wakeup the work thread itself and return 226 // Note: we hold the gate across the entire operation mainly for the 227 // benefit of our event sources so we can disable them cleanly. 228 closeGate(); 229 230 disableAllEventSources(); 231 232 is = IOSimpleLockLockDisableInterrupt(workToDoLock); 233 SETP(&fFlags, kLoopTerminate); 234 thread_wakeup_one((void *) &workToDo); 235 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 236 237 openGate(); 238 } 239 else /* !workThread */ { 240 IOEventSource *event, *next; 241 242 for (event = eventChain; event; event = next) { 243 next = event->getNext(); 244 event->setWorkLoop(0); 245 event->setNext(0); 246 event->release(); 247 } 248 eventChain = 0; 249 250 for (event = passiveEventChain; event; event = next) { 251 next = event->getNext(); 252 event->setWorkLoop(0); 253 event->setNext(0); 254 event->release(); 255 } 256 passiveEventChain = 0; 257 258 // Either we have a partial initialization to clean up 259 // or the workThread itself is performing hari-kari. 260 // Either way clean up all of our resources and return. 261 262 if (controlG) { 263 controlG->release(); 264 controlG = 0; 265 } 266 267 if (workToDoLock) { 268 IOSimpleLockFree(workToDoLock); 269 workToDoLock = 0; 270 } 271 272 if (gateLock) { 273 IORecursiveLockFree(gateLock); 274 gateLock = 0; 275 } 276 277 IOStatisticsUnregisterCounter(); 278 279 if (reserved) { 280 IODelete(reserved, ExpansionData, 1); 281 reserved = 0; 282 } 283 284 super::free(); 285 } 286} 287 288IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent) 289{ 290 return controlG->runCommand((void *) mAddEvent, (void *) newEvent); 291} 292 293IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove) 294{ 295 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove); 296} 297 298void IOWorkLoop::enableAllEventSources() const 299{ 300 IOEventSource *event; 301 302 for (event = eventChain; event; event = event->getNext()) 303 event->enable(); 304 305 for (event = passiveEventChain; event; event = event->getNext()) 306 event->enable(); 307} 308 309void IOWorkLoop::disableAllEventSources() const 310{ 311 IOEventSource *event; 312 313 for (event = eventChain; event; event = event->getNext()) 314 event->disable(); 315 316 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ 317 for (event = passiveEventChain; event; event = event->getNext()) 318 if (event != controlG) // Don't disable the control gate 319 event->disable(); 320} 321 322void IOWorkLoop::enableAllInterrupts() const 323{ 324 IOEventSource *event; 325 326 for (event = eventChain; event; event = event->getNext()) 327 if (OSDynamicCast(IOInterruptEventSource, event)) 328 event->enable(); 329} 330 331void IOWorkLoop::disableAllInterrupts() const 332{ 333 IOEventSource *event; 334 335 for (event = eventChain; event; event = event->getNext()) 336 if (OSDynamicCast(IOInterruptEventSource, event)) 337 event->disable(); 338} 339 340 341/* virtual */ bool IOWorkLoop::runEventSources() 342{ 343 bool res = false; 344 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; 345 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; 346 347 closeGate(); 348 if (ISSETP(&fFlags, kLoopTerminate)) 349 goto abort; 350 351 if (traceWL) 352 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 353 354 bool more; 355 do { 356 CLRP(&fFlags, kLoopRestart); 357 more = false; 358 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 359 workToDo = false; 360 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 361 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ 362 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { 363 364 if (traceES) 365 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 366 367 more |= evnt->checkForWork(); 368 369 if (traceES) 370 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 371 372 if (ISSETP(&fFlags, kLoopTerminate)) 373 goto abort; 374 else if (fFlags & kLoopRestart) { 375 more = true; 376 break; 377 } 378 } 379 } while (more); 380 381 res = true; 382 383 if (traceWL) 384 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 385 386abort: 387 openGate(); 388 return res; 389} 390 391/* virtual */ void IOWorkLoop::threadMain() 392{ 393restartThread: 394 do { 395 if ( !runEventSources() ) 396 goto exitThread; 397 398 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 399 if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { 400 assert_wait((void *) &workToDo, false); 401 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 402 thread_continue_t cptr = NULL; 403 if (!reserved || !(kPreciousStack & reserved->options)) 404 cptr = OSMemberFunctionCast( 405 thread_continue_t, this, &IOWorkLoop::threadMain); 406 thread_block_parameter(cptr, this); 407 goto restartThread; 408 /* NOTREACHED */ 409 } 410 411 // At this point we either have work to do or we need 412 // to commit suicide. But no matter 413 // Clear the simple lock and retore the interrupt state 414 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 415 } while(workToDo); 416 417exitThread: 418 thread_t thread = workThread; 419 workThread = 0; // Say we don't have a loop and free ourselves 420 free(); 421 422 thread_deallocate(thread); 423 (void) thread_terminate(thread); 424} 425 426IOThread IOWorkLoop::getThread() const 427{ 428 return workThread; 429} 430 431bool IOWorkLoop::onThread() const 432{ 433 return (IOThreadSelf() == workThread); 434} 435 436bool IOWorkLoop::inGate() const 437{ 438 return IORecursiveLockHaveLock(gateLock); 439} 440 441// Internal APIs used by event sources to control the thread 442void IOWorkLoop::signalWorkAvailable() 443{ 444 if (workToDoLock) { 445 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 446 workToDo = true; 447 thread_wakeup_one((void *) &workToDo); 448 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 449 } 450} 451 452void IOWorkLoop::openGate() 453{ 454 IOStatisticsOpenGate(); 455 IORecursiveLockUnlock(gateLock); 456} 457 458void IOWorkLoop::closeGate() 459{ 460 IORecursiveLockLock(gateLock); 461 IOStatisticsCloseGate(); 462} 463 464bool IOWorkLoop::tryCloseGate() 465{ 466 bool res = (IORecursiveLockTryLock(gateLock) != 0); 467 if (res) { 468 IOStatisticsCloseGate(); 469 } 470 return res; 471} 472 473int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) 474{ 475 int res; 476 IOStatisticsOpenGate(); 477 res = IORecursiveLockSleep(gateLock, event, interuptibleType); 478 IOStatisticsCloseGate(); 479 return res; 480} 481 482int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) 483{ 484 int res; 485 IOStatisticsOpenGate(); 486 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); 487 IOStatisticsCloseGate(); 488 return res; 489} 490 491void IOWorkLoop::wakeupGate(void *event, bool oneThread) 492{ 493 IORecursiveLockWakeup(gateLock, event, oneThread); 494} 495 496IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, 497 void *arg0, void *arg1, 498 void *arg2, void *arg3) 499{ 500 IOReturn res; 501 502 // closeGate is recursive so don't worry if we already hold the lock. 503 closeGate(); 504 res = (*inAction)(target, arg0, arg1, arg2, arg3); 505 openGate(); 506 507 return res; 508} 509 510IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) 511{ 512 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; 513 IOEventSource *inEvent = (IOEventSource *) inD; 514 IOReturn res = kIOReturnSuccess; 515 516 switch (command) 517 { 518 case mAddEvent: 519 if (!inEvent->getWorkLoop()) { 520 SETP(&fFlags, kLoopRestart); 521 522 inEvent->retain(); 523 inEvent->setWorkLoop(this); 524 inEvent->setNext(0); 525 526 /* Check if this is a passive or active event source being added */ 527 if (eventSourcePerformsWork(inEvent)) { 528 529 if (!eventChain) 530 eventChain = inEvent; 531 else { 532 IOEventSource *event, *next; 533 534 for (event = eventChain; (next = event->getNext()); event = next) 535 ; 536 event->setNext(inEvent); 537 538 } 539 540 } 541 else { 542 543 if (!passiveEventChain) 544 passiveEventChain = inEvent; 545 else { 546 IOEventSource *event, *next; 547 548 for (event = passiveEventChain; (next = event->getNext()); event = next) 549 ; 550 event->setNext(inEvent); 551 552 } 553 554 } 555 IOStatisticsAttachEventSource(); 556 } 557 break; 558 559 case mRemoveEvent: 560 if (inEvent->getWorkLoop()) { 561 IOStatisticsDetachEventSource(); 562 563 if (eventSourcePerformsWork(inEvent)) { 564 if (eventChain == inEvent) 565 eventChain = inEvent->getNext(); 566 else { 567 IOEventSource *event, *next; 568 569 event = eventChain; 570 while ((next = event->getNext()) && next != inEvent) 571 event = next; 572 573 if (!next) { 574 res = kIOReturnBadArgument; 575 break; 576 } 577 event->setNext(inEvent->getNext()); 578 } 579 } 580 else { 581 if (passiveEventChain == inEvent) 582 passiveEventChain = inEvent->getNext(); 583 else { 584 IOEventSource *event, *next; 585 586 event = passiveEventChain; 587 while ((next = event->getNext()) && next != inEvent) 588 event = next; 589 590 if (!next) { 591 res = kIOReturnBadArgument; 592 break; 593 } 594 event->setNext(inEvent->getNext()); 595 } 596 } 597 598 inEvent->setWorkLoop(0); 599 inEvent->setNext(0); 600 inEvent->release(); 601 SETP(&fFlags, kLoopRestart); 602 } 603 break; 604 605 default: 606 return kIOReturnUnsupported; 607 } 608 609 return res; 610} 611 612bool 613IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) 614{ 615 bool result = true; 616 617 /* 618 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). 619 * The assumption is that if you override checkForWork(), you need to be 620 * active and not passive. 621 * 622 * We picked a known quantity controlG that does not override 623 * IOEventSource::checkForWork(), namely the IOCommandGate associated with 624 * the workloop to which this event source is getting attached. 625 * 626 * We do a pointer comparison on the offset in the vtable for inNewEvent against 627 * the offset in the vtable for inReferenceEvent. This works because 628 * IOCommandGate's slot for checkForWork() has the address of 629 * IOEventSource::checkForWork() in it. 630 * 631 * Think of OSMemberFunctionCast yielding the value at the vtable offset for 632 * checkForWork() here. We're just testing to see if it's the same or not. 633 * 634 */ 635 if (controlG) { 636 void * ptr1; 637 void * ptr2; 638 639 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); 640 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); 641 642 if (ptr1 == ptr2) 643 result = false; 644 } 645 646 return result; 647} 648